Support multiple IP addresses for http_x_forwarded_for

.travis.yml

@@ -1,12 +1,14 @@
 language: go
 sudo: false
 go:
-  - 1.1
-  - 1.2
+  - 1.2.2
   - 1.3
   - 1.4
   - 1.5
+  - 1.12
+  - 1.13
   - tip
+go_import_path: github.com/satyrius/gonx
 install: make deps
 script:
   - go test -v -bench .

example/common/common.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	gonx "../.."
 	"flag"
 	"fmt"
+	gonx "github.com/satyrius/gonx"
 	"io"
 	"os"
 	"strings"

example/nginx/nginx.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	gonx "../.."
 	"flag"
 	"fmt"
+	gonx "github.com/satyrius/gonx"
 	"io"
 	"os"
 	"strings"

example/reduce/reduce.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	gonx "../.."
 	"flag"
 	"fmt"
+	gonx "github.com/satyrius/gonx"
 	"io"
 	"os"
 	"strings"

parser.go

@@ -19,6 +19,11 @@ type Parser struct {
 	regexp *regexp.Regexp
 }
 
+func getSpecialNginxRegexes() map[string]string {
+	return map[string]string{
+		"http_x_forwarded_for": `[^, ]*(?:, ?[^, ]+)*`}
+}
+
 // NewParser returns a new Parser, use given log format to create its internal
 // strings parsing regexp.
 func NewParser(format string) *Parser {
@@ -32,10 +37,21 @@ func NewParser(format string) *Parser {
 		)
 	}
 
-	// Second replace each fileds to regexp grouping
-	quotedFormat := regexp.QuoteMeta(preparedFormat + " ")
-	re := regexp.MustCompile(`\\\$([A-Za-z0-9_]+)(?:\\\$[A-Za-z0-9_])*(\\?([^$\\A-Za-z0-9_]))`).ReplaceAllString(
-		quotedFormat, "(?P<$1>[^$3]*)$2")
+	formatRegex := regexp.MustCompile(`([^$ ]*)\$([a-z_]+)([^$ ]*)([ ]?)`)
+	specialNginxRegexes := getSpecialNginxRegexes()
+	fields := formatRegex.FindAllStringSubmatch(preparedFormat+" ", -1)
+	re := formatRegex.ReplaceAllString(preparedFormat+" ", "$2$4")
+	for _, field := range fields {
+		terminateChar := field[3]
+		if len([]rune(terminateChar)) == 0 {
+			terminateChar = field[4]
+		}
+		if specialRegex, found := specialNginxRegexes[field[2]]; found {
+			re = strings.Replace(re, field[2]+field[4], regexp.QuoteMeta(field[1])+"(?P<"+field[2]+">"+specialRegex+")"+regexp.QuoteMeta(field[3])+field[4], 1)
+		} else {
+			re = strings.Replace(re, field[2]+field[4], regexp.QuoteMeta(field[1])+"(?P<"+field[2]+">[^"+terminateChar+"]*)"+regexp.QuoteMeta(field[3]+field[4]), 1)
+		}
+	}
 
 	// Finally remove placeholder
 	re = regexp.MustCompile(fmt.Sprintf(".%s", placeholder)).ReplaceAllString(re, "")

parser_test.go

@@ -9,7 +9,7 @@ import (
 func TestParser(t *testing.T) {
 	Convey("Test Parser", t, func() {
 		Convey("Parse custom format", func() {
-			format := "$remote_addr [$time_local] \"$request\" $status"
+			format := "$remote_addr [$time_local] \"$request\" $status $http_x_forwarded_for"
 			parser := NewParser(format)
 
 			Convey("Ensure parser format is ok", func() {
@@ -18,29 +18,31 @@ func TestParser(t *testing.T) {
 
 			Convey("Test format to regexp translation", func() {
 				So(parser.regexp.String(), ShouldEqual,
-					`^(?P<remote_addr>[^ ]*) \[(?P<time_local>[^]]*)\] "(?P<request>[^"]*)" (?P<status>[^ ]*)`)
+					`^(?P<remote_addr>[^ ]*) \[(?P<time_local>[^]]*)\] "(?P<request>[^"]*)" (?P<status>[^ ]*) (?P<http_x_forwarded_for>[^, ]*(?:, ?[^, ]+)*)`)
 			})
 
 			Convey("ParseString", func() {
-				line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "GET /api/foo/bar HTTP/1.1" 200`
+				line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "GET /api/foo/bar HTTP/1.1" 200 1.2.3.4,5.6.7.8, 9.10.11.12`
 				expected := NewEntry(Fields{
-					"remote_addr": "89.234.89.123",
-					"time_local":  "08/Nov/2013:13:39:18 +0000",
-					"request":     "GET /api/foo/bar HTTP/1.1",
-					"status":      "200",
+					"remote_addr":          "89.234.89.123",
+					"time_local":           "08/Nov/2013:13:39:18 +0000",
+					"request":              "GET /api/foo/bar HTTP/1.1",
+					"status":               "200",
+					"http_x_forwarded_for": "1.2.3.4,5.6.7.8, 9.10.11.12",
 				})
 				entry, err := parser.ParseString(line)
 				So(err, ShouldBeNil)
 				So(entry, ShouldResemble, expected)
 			})
 
 			Convey("Handle empty values", func() {
-				line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "" 200`
+				line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "" 200 1.2.3.4`
 				expected := NewEntry(Fields{
-					"remote_addr": "89.234.89.123",
-					"time_local":  "08/Nov/2013:13:39:18 +0000",
-					"request":     "",
-					"status":      "200",
+					"remote_addr":          "89.234.89.123",
+					"time_local":           "08/Nov/2013:13:39:18 +0000",
+					"request":              "",
+					"status":               "200",
+					"http_x_forwarded_for": "1.2.3.4",
 				})
 				entry, err := parser.ParseString(line)
 				So(err, ShouldBeNil)