diff --git a/parser/user_traffic.go b/parser/user_traffic.go deleted file mode 100644 index 76df9fe..0000000 --- a/parser/user_traffic.go +++ /dev/null @@ -1,150 +0,0 @@ -package parser - -import ( - "fmt" - "strconv" - "strings" - "time" -) - -//UserTraffic is a single decoded user traffic log line -type UserTraffic struct { - Status int `json:"status"` - RequestSize int64 `json:"request_size"` - ResponseSize int64 `json:"response_size"` - Timing int64 `json:"timing"` - Timestamp time.Time `json:"timestamp"` - RequestID string `json:"request_id"` - Result string `json:"result"` - CSID string `json:"csid"` - CCID string `jsond:"ccid"` - CID string `json:"cid"` - Proto string `json:"proto"` - Method string `json:"method"` - URL string `json:"url"` - SID string `json:"sid"` - AID string `json:"aid"` - DID string `json:"did"` - Cancel string `json:"cancel"` - CCancel string `json:"ccancel"` - ProxyType string `json:"proxy_type"` - FID string `json:"fid"` - ContentType string `json:"content_type"` - Address string `json:"address"` - Country string `json:"country"` - Referrer string `json:"referrer"` - CW string `json:"cw"` - SSLVersion string `json:"ssl_version"` - SSLCipher string `json:"ssl_cipher"` - ENC string `json:"enc"` - EFS int `json:"efs"` - UserAgent string `json:"ua"` - Unparsed []string `json:"unparsed"` -} - -//ParseUserTrafficRecord parses a raw user traffic log line into a UserTraffic struct -//A slice of any unknown kv pairs or unbalanced fields will be appended to the UserTraffic -//structs Unparsed field. -func ParseUserTrafficRecord(raw string) (*UserTraffic, error) { - var ut UserTraffic - var err error - - if count := strings.Count(raw, `@timestamp`); count > 1 { - return nil, fmt.Errorf("%d @timestamp fields detected", count) - } - - praw := strings.SplitN(raw, " ua=", 2) - - if len(praw) > 1 { - ut.UserAgent = praw[1] - } - - for _, field := range strings.Fields(praw[0]) { - parts := strings.SplitN(field, "=", 2) - if len(parts) != 2 { // most commonly due to kv's with duplicate value fields - ut.Unparsed = append(ut.Unparsed, field) - continue - } - switch parts[0] { - case "request_id": - ut.RequestID = parts[1] - case "@timestamp", "timestamp": - tsFloat, err := strconv.ParseFloat(parts[1], 64) - if err != nil { - return nil, fmt.Errorf("malformed field (%s) value: %s", parts[0], parts[1]) - } - ut.Timestamp = time.Unix(int64(tsFloat), 0) - case "timing": - if ut.Timing, err = strconv.ParseInt(parts[1], 10, 64); err != nil { - return nil, fmt.Errorf("malformed field (%s) value: %s", parts[0], parts[1]) - } - case "result": - ut.Result = parts[1] - case "csid": - ut.CSID = strings.TrimSuffix(parts[1], ",") - case "cid": - ut.CID = strings.TrimSuffix(parts[1], ",") - case "ccid": - ut.CCID = strings.TrimSuffix(parts[1], ",") - case "status": - if ut.Status, err = strconv.Atoi(parts[1]); err != nil { - return nil, fmt.Errorf("malformed field (%s) value: %s", parts[0], parts[1]) - } - case "request_size": - if ut.RequestSize, err = strconv.ParseInt(parts[1], 10, 64); err != nil { - return nil, fmt.Errorf("malformed field (%s) value: %s", parts[0], parts[1]) - } - case "response_size": - if ut.ResponseSize, err = strconv.ParseInt(parts[1], 10, 64); err != nil { - return nil, fmt.Errorf("malformed field (%s) value: %s", parts[0], parts[1]) - } - case "proto": - ut.Proto = parts[1] - case "method": - ut.Method = parts[1] - case "url": - ut.URL = parts[1] - case "sid": - //SID's & AID's somewhat frequently have a trailing comma, while we try not to manipulate - //or clean up the log lines inline this was an easy one that we felt we should - //proactively handle - ut.SID = strings.TrimSuffix(parts[1], ",") - case "aid": - //SID's & AID's somewhat frequently have a trailing comma, while we try not to manipulate - //or clean up the log lines inline this was an easy one that we felt we should - //proactively handle - ut.AID = strings.TrimSuffix(parts[1], ",") - case "did": - ut.DID = strings.TrimSuffix(parts[1], ",") - case "cancel": - ut.Cancel = strings.TrimSuffix(parts[1], ",") - case "proxy_type": - ut.ProxyType = strings.TrimSuffix(parts[1], ",") - case "fid": - ut.FID = strings.TrimSuffix(parts[1], ",") - case "content_type": - ut.ContentType = strings.TrimSuffix(parts[1], ",") - case "address": - ut.Address = strings.TrimSuffix(parts[1], ",") - case "country": - ut.Country = strings.TrimSuffix(parts[1], ",") - case "referrer": - ut.Referrer = strings.TrimSuffix(parts[1], ",") - case "cw": - ut.CW = strings.TrimSuffix(parts[1], ",") - case "ssl_version": - ut.SSLVersion = strings.TrimSuffix(parts[1], ",") - case "ssl_cipher": - ut.SSLCipher = strings.TrimSuffix(parts[1], ",") - case "enc": - ut.ENC = strings.TrimSuffix(parts[1], ",") - case "efs": - if ut.EFS, err = strconv.Atoi(parts[1]); err != nil { - return nil, fmt.Errorf("malformed field (%s) value: %s", parts[0], parts[1]) - } - default: - ut.Unparsed = append(ut.Unparsed, field) - } - } - return &ut, nil -} diff --git a/parser/user_traffic_test.go b/parser/user_traffic_test.go deleted file mode 100644 index 3de4115..0000000 --- a/parser/user_traffic_test.go +++ /dev/null @@ -1,272 +0,0 @@ -package parser - -import ( - "bytes" - "testing" - "text/template" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -//raw record to test against as a fail-safe (incase the template drifts) -var rawUTRecord = "request_id=c9948493-1ece-4d21-a2d1-f96a9feded3c @timestamp=1585844380.949 timing=1 result=TCP_MEM_HIT cid=- ccid=12345 status=200 request_size=1 response_size=66000 proto=http/2 method=GET url=http://localhost/something/1591294965428966000/something.jpg sid=18bb190b-6727-497a-af8b-f03287d14caf, aid=1591294965428966000 did=5e85df2043933dd053ebec6f cancel=- proxy_type=- stuff=things oneother=\"onething\" fid=- content_type=text/plain address=2605:6000:1714:56e:c98a:445c:febd:6baf country=US referrer=localhost cw=- ssl_version=TLSv1.2 ssl_cipher=ECDHE-RSA-AES256-GCM-SHA384 enc=- efs=1 ua=Mozilla/5.0 (X11; CrOS x86_64 12239.92.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.136 Safari/537.36" - -var utLineTemplateStr = "request_id={{.requestIDField}} " + - "@timestamp={{.atTimestampField}} " + - "timing={{.timingField}} " + - "result={{.resultField}} " + - "cid={{.cidField}} " + - "ccid={{.ccidField}} " + - "status={{.statusField}} " + - "request_size={{.requestSizeField}} " + - "response_size={{.responseSizeField}} " + - "proto={{.protoField}} " + - "method={{.methodField}} " + - "url={{.urlField}} " + - "sid={{.sidField}} " + - "aid={{.aidField}} " + - "did={{.didField}} " + - "cancel={{.cancelField}} " + - "proxy_type={{.proxyTypeField}} " + - "{{.extraFields}} " + //extra fields we should gracefully handle by adding to the others map - "fid={{.fidField}} " + - "content_type={{.contentTypeField}} " + - "address={{.addressField}} " + - "country={{.countryField}} " + - "referrer={{.referrerField}} " + - "cw={{.cwField}} " + - "ssl_version={{.sslVersionField}} " + - "ssl_cipher={{.sslCipherField}} " + - "enc={{.encField}} " + - "efs={{.efs}} " + - "ua={{.uaField}}" - -var utLineTemplate = template.Must(template.New("user_traffic").Parse(utLineTemplateStr)) - -var ( - extraFields = "stuff=things oneother=\"onething\"" - requestIDField = "c9948493-1ece-4d21-a2d1-f96a9feded3c" - atTimestampField = "1585844380.949" - timingField = "1" - resultField = "TCP_MEM_HIT" - cidField = "-" - ccidField = "12345" - statusField = "200" - requestSizeField = "1" - responseSizeField = "66000" - protoField = "http/2" - methodField = "GET" - urlField = "http://localhost/something/1591294965428966000/something.jpg" - sidField = "18bb190b-6727-497a-af8b-f03287d14caf" - aidField = "1591294965428966000" - didField = "5e85df2043933dd053ebec6f" - cancelField = "-" - proxyTypeField = "-" - fidField = "-" - contentTypeField = "text/plain" - addressField = "2605:6000:1714:56e:c98a:445c:febd:6baf" - countryField = "US" - referrerField = "localhost" - cwField = "-" - uaField = "Mozilla/5.0 (X11; CrOS x86_64 12239.92.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.136 Safari/537.36" - sslVersionField = "TLSv1.2" - sslCipherField = "ECDHE-RSA-AES256-GCM-SHA384" - encField = "-" - efs = "1" -) - -func defaultValues() map[string]string { - return map[string]string{ - "extraFields": extraFields, - "requestIDField": requestIDField, - "atTimestampField": atTimestampField, - "timingField": timingField, - "resultField": resultField, - "cidField": cidField, - "ccidField": ccidField, - "statusField": statusField, - "requestSizeField": requestSizeField, - "responseSizeField": responseSizeField, - "protoField": protoField, - "methodField": methodField, - "urlField": urlField, - "sidField": sidField, - "aidField": aidField, - "didField": didField, - "cancelField": cancelField, - "proxyTypeField": proxyTypeField, - "fidField": fidField, - "contentTypeField": contentTypeField, - "addressField": addressField, - "countryField": countryField, - "referrerField": referrerField, - "cwField": cwField, - "uaField": uaField, - "sslVersionField": sslVersionField, - "sslCipherField": sslCipherField, - "encField": encField, - "efs": efs, - } -} - -func genUserTrafficLine(t *testing.T, values map[string]string) string { - buf := new(bytes.Buffer) - require.NoError(t, utLineTemplate.Execute(buf, values)) - return buf.String() -} - -func TestParseUserTrafficPayload(t *testing.T) { - expected := &UserTraffic{ - Status: 200, - RequestSize: 1, - ResponseSize: 66000, - Timing: 1, - Timestamp: time.Unix(int64(1585844380), 0), - RequestID: "c9948493-1ece-4d21-a2d1-f96a9feded3c", - Result: "TCP_MEM_HIT", - CSID: "", - CID: "-", - CCID: "12345", - Proto: "http/2", - Method: "GET", - URL: "http://localhost/something/1591294965428966000/something.jpg", - SID: "18bb190b-6727-497a-af8b-f03287d14caf", - AID: "1591294965428966000", - DID: "5e85df2043933dd053ebec6f", - Cancel: "-", - CCancel: "", - ProxyType: "-", - FID: "-", - ContentType: "text/plain", - Address: "2605:6000:1714:56e:c98a:445c:febd:6baf", - Country: "US", - Referrer: "localhost", - SSLCipher: "ECDHE-RSA-AES256-GCM-SHA384", - SSLVersion: "TLSv1.2", - ENC: "-", - CW: "-", - EFS: 1, - UserAgent: "Mozilla/5.0 (X11; CrOS x86_64 12239.92.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.136 Safari/537.36", - Unparsed: []string{"stuff=things", "oneother=\"onething\""}, - } - - ut, err := ParseUserTrafficRecord(genUserTrafficLine(t, defaultValues())) - require.NoError(t, err) - assert.Equal(t, expected, ut) - - // quick fail safe - we should *always* be able to snag a raw log line and parse out these key fields. - failSafeUT := &UserTraffic{ - Status: 200, - RequestSize: 1, - ResponseSize: 66000, - Timing: 1, - Timestamp: time.Unix(int64(1585844380), 0), - RequestID: "c9948493-1ece-4d21-a2d1-f96a9feded3c", - URL: "http://localhost/something/1591294965428966000/something.jpg", - SID: "18bb190b-6727-497a-af8b-f03287d14caf", - AID: "1591294965428966000", - DID: "5e85df2043933dd053ebec6f", - Address: "2605:6000:1714:56e:c98a:445c:febd:6baf", - } - - ut, err = ParseUserTrafficRecord(rawUTRecord) - require.NoError(t, err) - assert.Equal(t, failSafeUT.Status, ut.Status) - assert.Equal(t, failSafeUT.RequestSize, ut.RequestSize) - assert.Equal(t, failSafeUT.ResponseSize, ut.ResponseSize) - assert.Equal(t, failSafeUT.Timing, ut.Timing) - assert.Equal(t, failSafeUT.Timestamp, ut.Timestamp) - assert.Equal(t, failSafeUT.URL, ut.URL) - assert.Equal(t, failSafeUT.SID, ut.SID) - assert.Equal(t, failSafeUT.AID, ut.AID) - assert.Equal(t, failSafeUT.DID, ut.DID) - assert.Equal(t, failSafeUT.Address, ut.Address) - assert.Len(t, ut.Unparsed, 2) - assert.Contains(t, ut.Unparsed, "stuff=things") - assert.Contains(t, ut.Unparsed, "oneother=\"onething\"") -} - -func TestSidWithComma(t *testing.T) { - fields := defaultValues() - fields["sidField"] = fields["sidField"] + "," - ut, err := ParseUserTrafficRecord(genUserTrafficLine(t, fields)) - require.NoError(t, err) - assert.Equal(t, sidField, ut.SID) -} - -func TestAidWithComma(t *testing.T) { - fields := defaultValues() - fields["aidField"] = fields["aidField"] + "," - ut, err := ParseUserTrafficRecord(genUserTrafficLine(t, fields)) - require.NoError(t, err) - assert.Equal(t, sidField, ut.SID) -} - -func TestErrOnExtraTimestamp(t *testing.T) { - withExtraTimestamp := "@timestamp=181818181 " + genUserTrafficLine(t, defaultValues()) - ut, err := ParseUserTrafficRecord(withExtraTimestamp) - require.Error(t, err) - require.Nil(t, ut) -} - -func TestKeyWithNoValue(t *testing.T) { - keyMissingValue := "randomKeyWithNoValue " + genUserTrafficLine(t, defaultValues()) - ut, err := ParseUserTrafficRecord(keyMissingValue) - require.NoError(t, err) - require.Len(t, ut.Unparsed, 3) - require.Contains(t, ut.Unparsed, "randomKeyWithNoValue") -} - -func TestMalformedTimestamp(t *testing.T) { - fields := defaultValues() - fields["atTimestampField"] = "time_is_relative" - ut, err := ParseUserTrafficRecord(genUserTrafficLine(t, fields)) - require.Error(t, err) - require.Nil(t, ut) -} - -func TestMalformedTiming(t *testing.T) { - fields := defaultValues() - fields["timingField"] = "somestring" - ut, err := ParseUserTrafficRecord(genUserTrafficLine(t, fields)) - require.Error(t, err) - require.Nil(t, ut) -} - -func TestMalformedStatus(t *testing.T) { - fields := defaultValues() - fields["statusField"] = "somestring" - ut, err := ParseUserTrafficRecord(genUserTrafficLine(t, fields)) - require.Error(t, err) - require.Nil(t, ut) -} - -func TestMalformedRequestSize(t *testing.T) { - fields := defaultValues() - fields["requestSizeField"] = "somestring" - ut, err := ParseUserTrafficRecord(genUserTrafficLine(t, fields)) - require.Error(t, err) - require.Nil(t, ut) -} - -func TestMalformedResponseSize(t *testing.T) { - fields := defaultValues() - fields["responseSizeField"] = "somestring" - ut, err := ParseUserTrafficRecord(genUserTrafficLine(t, fields)) - require.Error(t, err) - require.Nil(t, ut) -} - -func TestDuplicateValueFields(t *testing.T) { - fields := defaultValues() - fields["countryField"] = "US, US" - ut, err := ParseUserTrafficRecord(genUserTrafficLine(t, fields)) - require.NoError(t, err) - require.Equal(t, "US", ut.Country) - require.Len(t, ut.Unparsed, 3) - require.Contains(t, ut.Unparsed, "stuff=things") - require.Contains(t, ut.Unparsed, "oneother=\"onething\"") - require.Contains(t, ut.Unparsed, "US") -}