From 9864dae1ae30f03bb91c1d6f7ff706ce3e18838b Mon Sep 17 00:00:00 2001 From: "Derrick J. Wippler" Date: Tue, 20 Aug 2019 09:44:17 -0500 Subject: [PATCH 1/2] Added JSONFormater to project --- common/common.go | 23 ++++++++ formatter.go | 67 +++++++++++++++++++++++ go.mod | 23 ++++++++ go.sum | 117 +++++++++++++++++++++++++++++++++++++++++ kafkahook/kafkahook.go | 24 +-------- 5 files changed, 231 insertions(+), 23 deletions(-) create mode 100644 formatter.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/common/common.go b/common/common.go index 156afff..75a7bcf 100644 --- a/common/common.go +++ b/common/common.go @@ -1,9 +1,11 @@ package common import ( + "bufio" "encoding/json" "fmt" "net/http" + "os" "runtime" "strings" @@ -102,3 +104,24 @@ func ToFields(items map[string]string) logrus.Fields { } return result } + +func GetDockerCID() string { + f, err := os.Open("/proc/self/cgroup") + if err != nil { + return "" + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Split(line, "/docker/") + if len(parts) != 2 { + continue + } + + fullDockerCID := parts[1] + return fullDockerCID[:12] + } + return "" +} \ No newline at end of file diff --git a/formatter.go b/formatter.go new file mode 100644 index 0000000..de5efc6 --- /dev/null +++ b/formatter.go @@ -0,0 +1,67 @@ +package logrus_hooks + +import ( + "os" + "path/filepath" + "strings" + + "github.com/mailgun/holster/errors" + "github.com/mailgun/holster/stack" + "github.com/mailgun/logrus-hooks/common" + "github.com/mailru/easyjson/jwriter" + "github.com/sirupsen/logrus" +) + +type JSONFormater struct { + appName string + hostName string + cid string + pid int +} + +func NewJSONFormater() *JSONFormater { + f := JSONFormater{} + var err error + + if f.hostName, err = os.Hostname(); err != nil { + f.hostName = "unknown" + } + f.appName = filepath.Base(os.Args[0]) + if f.pid = os.Getpid(); f.pid == 1 { + f.pid = 0 + } + f.cid = common.GetDockerCID() + return &f +} + + +func (f *JSONFormater) Format(entry *logrus.Entry) ([]byte, error) { + var caller *stack.FrameInfo + + caller = common.GetLogrusCaller() + + rec := &common.LogRecord{ + Category: "logrus", + AppName: f.appName, + HostName: f.hostName, + LogLevel: strings.ToUpper(entry.Level.String()), + FileName: caller.File, + FuncName: caller.Func, + LineNo: caller.LineNo, + Message: entry.Message, + Context: nil, + Timestamp: common.Number(float64(entry.Time.UnixNano()) / 1000000000), + CID: f.cid, + PID: f.pid, + } + rec.FromFields(entry.Data) + + var w jwriter.Writer + rec.MarshalEasyJSON(&w) + if w.Error != nil { + return nil, errors.Wrap(w.Error, "while marshalling json") + } + buf := w.Buffer.BuildBytes() + + return buf, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d1cc32d --- /dev/null +++ b/go.mod @@ -0,0 +1,23 @@ +module github.com/mailgun/logrus-hooks + +go 1.12 + +require ( + github.com/Shopify/sarama v1.23.1 + github.com/ahmetb/go-linq v3.0.0+incompatible // indirect + github.com/fatih/structs v1.1.0 // indirect + github.com/go-ini/ini v1.46.0 // indirect + github.com/kr/pretty v0.1.0 // indirect + github.com/mailgun/holster v3.0.0+incompatible + github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e + github.com/onsi/ginkgo v1.9.0 // indirect + github.com/onsi/gomega v1.6.0 // indirect + github.com/pkg/errors v0.8.1 + github.com/sirupsen/logrus v1.4.2 + github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect + github.com/spf13/cast v1.3.0 // indirect + github.com/thrawn01/args v0.3.0 + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 + gopkg.in/ini.v1 v1.46.0 // indirect + gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..56de7bb --- /dev/null +++ b/go.sum @@ -0,0 +1,117 @@ +github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 h1:2T/jmrHeTezcCM58lvEQXs0UpQJCo5SoGAcg+mbSTIg= +github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Shopify/sarama v1.23.1 h1:XxJBCZEoWJtoWjf/xRbmGUpAmTZGnuuF0ON0EvxxBrs= +github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= +github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/ahmetb/go-linq v3.0.0+incompatible h1:qQkjjOXKrKOTy83X8OpRmnKflXKQIL/mC/gMVVDMhOA= +github.com/ahmetb/go-linq v3.0.0+incompatible/go.mod h1:PFffvbdbtw+QTB0WKRP0cNht7vnCfnGlEpak/DVg5cY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-ini/ini v1.46.0 h1:hDJFfs/9f75875scvqLkhNB5Jz5/DybKEOZ5MLF+ng4= +github.com/go-ini/ini v1.46.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM= +github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailgun/holster v3.0.0+incompatible h1:bpt8ZCwLBrzjqfBZ5mobNb2NjesNeDHmsOO++Ek9Swc= +github.com/mailgun/holster v3.0.0+incompatible/go.mod h1:crzolGx27RP/IBT/BnPQiYBB9igmAFHGRrz0zlMP0b0= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc= +github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M= +github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41 h1:GeinFsrjWz97fAxVUEd748aV0cYL+I6k44gFJTCVvpU= +github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/thrawn01/args v0.3.0 h1:XbMnfGaw6nFbm8hgSncHu20cGrZMTP8BnxiusA43AeE= +github.com/thrawn01/args v0.3.0/go.mod h1:TnRiOFjyh7Wa6oC8ACFPc7KIvbzCiluphA3mJUiPIEo= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.46.0 h1:VeDZbLYGaupuvIrsYCEOe/L/2Pcs5n7hdO1ZTjporag= +gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.2.3 h1:hHMV/yKPwMnJhPuPx7pH2Uw/3Qyf+thJYlisUc44010= +gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/kafkahook/kafkahook.go b/kafkahook/kafkahook.go index 33b4728..7dae473 100644 --- a/kafkahook/kafkahook.go +++ b/kafkahook/kafkahook.go @@ -1,7 +1,6 @@ package kafkahook import ( - "bufio" "bytes" "context" "fmt" @@ -116,7 +115,7 @@ func New(conf Config) (*KafkaHook, error) { if h.pid = os.Getpid(); h.pid == 1 { h.pid = 0 } - h.cid = getDockerCID() + h.cid = common.GetDockerCID() return &h, nil } @@ -209,24 +208,3 @@ func (h *KafkaHook) Close() error { }) return err } - -func getDockerCID() string { - f, err := os.Open("/proc/self/cgroup") - if err != nil { - return "" - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - parts := strings.Split(line, "/docker/") - if len(parts) != 2 { - continue - } - - fullDockerCID := parts[1] - return fullDockerCID[:12] - } - return "" -} From d0db216e81723f1ba479a9d9ebe716695e622223 Mon Sep 17 00:00:00 2001 From: "Derrick J. Wippler" Date: Wed, 21 Aug 2019 10:07:56 -0500 Subject: [PATCH 2/2] All hooks now use JSONFormatter to consolidate code --- cmd/udplog/main.go | 2 +- common/common.go | 2 +- common/formatter.go | 67 +++++++++++++++++++++++++++++++++ formatter.go | 66 ++------------------------------ formatter_test.go | 38 +++++++++++++++++++ go.mod | 1 + go.sum | 2 + kafkahook/kafkahook.go | 53 ++++---------------------- kafkahook/kafkahook_test.go | 18 ++++----- levelfilter/levelfilter_test.go | 8 ++-- udploghook/udploghook.go | 50 +++++------------------- udploghook/udploghook_test.go | 18 ++++----- 12 files changed, 153 insertions(+), 172 deletions(-) create mode 100644 common/formatter.go create mode 100644 formatter_test.go diff --git a/cmd/udplog/main.go b/cmd/udplog/main.go index 043d4c8..9108a19 100644 --- a/cmd/udplog/main.go +++ b/cmd/udplog/main.go @@ -48,7 +48,7 @@ func main() { parts := strings.Split(opts.String("address"), ":") if len(parts) != 2 { - fmt.Fprint(os.Stderr, "address '%s' invalid must be in format 'host:port'", opts.String("address")) + fmt.Fprintf(os.Stderr, "address '%s' invalid must be in format 'host:port'", opts.String("address")) os.Exit(1) } diff --git a/common/common.go b/common/common.go index 75a7bcf..f0674fb 100644 --- a/common/common.go +++ b/common/common.go @@ -124,4 +124,4 @@ func GetDockerCID() string { return fullDockerCID[:12] } return "" -} \ No newline at end of file +} diff --git a/common/formatter.go b/common/formatter.go new file mode 100644 index 0000000..3936b1c --- /dev/null +++ b/common/formatter.go @@ -0,0 +1,67 @@ +package common + +import ( + "github.com/mailgun/holster/stack" + "github.com/mailru/easyjson/jwriter" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "strings" +) + +var DefaultFormatter = NewJSONFormater() + +func NewJSONFormater() *JSONFormater { + f := JSONFormater{} + var err error + + if f.hostName, err = os.Hostname(); err != nil { + f.hostName = "unknown" + } + f.appName = filepath.Base(os.Args[0]) + if f.pid = os.Getpid(); f.pid == 1 { + f.pid = 0 + } + f.cid = GetDockerCID() + return &f +} + + +func (f *JSONFormater) Format(entry *logrus.Entry) ([]byte, error) { + var caller *stack.FrameInfo + + caller = GetLogrusCaller() + + rec := &LogRecord{ + Category: "logrus", + AppName: f.appName, + HostName: f.hostName, + LogLevel: strings.ToUpper(entry.Level.String()), + FileName: caller.File, + FuncName: caller.Func, + LineNo: caller.LineNo, + Message: entry.Message, + Context: nil, + Timestamp: Number(float64(entry.Time.UnixNano()) / 1000000000), + CID: f.cid, + PID: f.pid, + } + rec.FromFields(entry.Data) + + var w jwriter.Writer + rec.MarshalEasyJSON(&w) + if w.Error != nil { + return nil, errors.Wrap(w.Error, "while marshalling json") + } + buf := w.Buffer.BuildBytes() + + return buf, nil +} + +type JSONFormater struct { + appName string + hostName string + cid string + pid int +} diff --git a/formatter.go b/formatter.go index de5efc6..577a231 100644 --- a/formatter.go +++ b/formatter.go @@ -1,67 +1,7 @@ package logrus_hooks -import ( - "os" - "path/filepath" - "strings" +import "github.com/mailgun/logrus-hooks/common" - "github.com/mailgun/holster/errors" - "github.com/mailgun/holster/stack" - "github.com/mailgun/logrus-hooks/common" - "github.com/mailru/easyjson/jwriter" - "github.com/sirupsen/logrus" -) - -type JSONFormater struct { - appName string - hostName string - cid string - pid int -} - -func NewJSONFormater() *JSONFormater { - f := JSONFormater{} - var err error - - if f.hostName, err = os.Hostname(); err != nil { - f.hostName = "unknown" - } - f.appName = filepath.Base(os.Args[0]) - if f.pid = os.Getpid(); f.pid == 1 { - f.pid = 0 - } - f.cid = common.GetDockerCID() - return &f -} - - -func (f *JSONFormater) Format(entry *logrus.Entry) ([]byte, error) { - var caller *stack.FrameInfo - - caller = common.GetLogrusCaller() - - rec := &common.LogRecord{ - Category: "logrus", - AppName: f.appName, - HostName: f.hostName, - LogLevel: strings.ToUpper(entry.Level.String()), - FileName: caller.File, - FuncName: caller.Func, - LineNo: caller.LineNo, - Message: entry.Message, - Context: nil, - Timestamp: common.Number(float64(entry.Time.UnixNano()) / 1000000000), - CID: f.cid, - PID: f.pid, - } - rec.FromFields(entry.Data) - - var w jwriter.Writer - rec.MarshalEasyJSON(&w) - if w.Error != nil { - return nil, errors.Wrap(w.Error, "while marshalling json") - } - buf := w.Buffer.BuildBytes() - - return buf, nil +func NewJSONFormater() *common.JSONFormater { + return common.DefaultFormatter } diff --git a/formatter_test.go b/formatter_test.go new file mode 100644 index 0000000..abd4400 --- /dev/null +++ b/formatter_test.go @@ -0,0 +1,38 @@ +package logrus_hooks_test + +import ( + "bufio" + "bytes" + "fmt" + hooks "github.com/mailgun/logrus-hooks" + "github.com/mailgun/logrus-hooks/common" + "github.com/mailru/easyjson" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "testing" +) + + +func TestNewJSONFormater(t *testing.T) { + var log = logrus.New() + var b bytes.Buffer + output := bufio.NewWriter(&b) + log.SetOutput(output) + + log.SetFormatter(hooks.NewJSONFormater()) + + log.Info("This is a test message") + + output.Flush() + fmt.Println(b.String()) + + rec := common.LogRecord{} + + err := easyjson.Unmarshal(b.Bytes(), &rec) + assert.Nil(t, err) + + assert.Equal(t, "This is a test message", rec.Message) + assert.Equal(t, "INFO", rec.LogLevel) + assert.Contains(t, rec.FuncName, "TestNewJSONFormater") + assert.Equal(t, "logrus", rec.Category) +} diff --git a/go.mod b/go.mod index d1cc32d..eab87b9 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/sirupsen/logrus v1.4.2 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect github.com/spf13/cast v1.3.0 // indirect + github.com/stretchr/testify v1.3.0 github.com/thrawn01/args v0.3.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 gopkg.in/ini.v1 v1.46.0 // indirect diff --git a/go.sum b/go.sum index 56de7bb..e3ddef6 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mailgun/holster v3.0.0+incompatible h1:bpt8ZCwLBrzjqfBZ5mobNb2NjesNeDHmsOO++Ek9Swc= github.com/mailgun/holster v3.0.0+incompatible/go.mod h1:crzolGx27RP/IBT/BnPQiYBB9igmAFHGRrz0zlMP0b0= +github.com/mailgun/logrus-hooks v1.2.1 h1:1paOVAsjXbvqoI1dft6gG+8WRc+CPJtnJjOjxhBkmuo= +github.com/mailgun/logrus-hooks v1.2.1/go.mod h1:YRm/XRR8Kmhtyq6FbqszTPq+RxEHvtc1Dlf0Beo8P1k= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/kafkahook/kafkahook.go b/kafkahook/kafkahook.go index 7dae473..2136609 100644 --- a/kafkahook/kafkahook.go +++ b/kafkahook/kafkahook.go @@ -4,18 +4,16 @@ import ( "bytes" "context" "fmt" + "github.com/mailgun/holster" + "github.com/mailgun/logrus-hooks/common" "io" "os" - "path/filepath" "strings" "sync" "time" "github.com/Shopify/sarama" "github.com/mailgun/holster/errors" - "github.com/mailgun/holster/stack" - "github.com/mailgun/logrus-hooks/common" - "github.com/mailru/easyjson/jwriter" "github.com/sirupsen/logrus" ) @@ -26,12 +24,6 @@ type KafkaHook struct { conf Config debug bool - // Process identity metadata - hostName string - appName string - cid string - pid int - // Sync stuff wg sync.WaitGroup once sync.Once @@ -41,6 +33,7 @@ type Config struct { Endpoints []string Topic string Producer sarama.AsyncProducer + Formatter logrus.Formatter } func NewWithContext(ctx context.Context, conf Config) (hook *KafkaHook, err error) { @@ -60,6 +53,8 @@ func NewWithContext(ctx context.Context, conf Config) (hook *KafkaHook, err erro } func New(conf Config) (*KafkaHook, error) { + // If no formatter defined, use the default + holster.SetDefault(&conf.Formatter, common.DefaultFormatter) var err error kafkaConfig := sarama.NewConfig() @@ -107,46 +102,14 @@ func New(conf Config) (*KafkaHook, error) { } } }() - - if h.hostName, err = os.Hostname(); err != nil { - h.hostName = "unknown" - } - h.appName = filepath.Base(os.Args[0]) - if h.pid = os.Getpid(); h.pid == 1 { - h.pid = 0 - } - h.cid = common.GetDockerCID() return &h, nil } func (h *KafkaHook) Fire(entry *logrus.Entry) error { - var caller *stack.FrameInfo - var err error - - caller = common.GetLogrusCaller() - - rec := &common.LogRecord{ - Category: "logrus", - AppName: h.appName, - HostName: h.hostName, - LogLevel: strings.ToUpper(entry.Level.String()), - FileName: caller.File, - FuncName: caller.Func, - LineNo: caller.LineNo, - Message: entry.Message, - Context: nil, - Timestamp: common.Number(float64(entry.Time.UnixNano()) / 1000000000), - CID: h.cid, - PID: h.pid, - } - rec.FromFields(entry.Data) - - var w jwriter.Writer - rec.MarshalEasyJSON(&w) - if w.Error != nil { - return errors.Wrap(w.Error, "while marshalling json") + buf, err := h.conf.Formatter.Format(entry) + if err != nil { + return errors.Wrap(err, "while formatting entry") } - buf := w.Buffer.BuildBytes() if h.debug { fmt.Printf("%s\n", string(buf)) diff --git a/kafkahook/kafkahook_test.go b/kafkahook/kafkahook_test.go index 20f737f..a5ee9bd 100644 --- a/kafkahook/kafkahook_test.go +++ b/kafkahook/kafkahook_test.go @@ -59,8 +59,8 @@ func (s *KafkaHookTests) TestKafkaHookINFO(c *C) { c.Assert(req["message"], Equals, "this is a test") c.Assert(req["context"], Equals, nil) c.Assert(req["logLevel"], Equals, "INFO") - c.Assert(strings.HasSuffix(req["filename"].(string), - "github.com/mailgun/logrus-hooks/kafkahook/kafkahook_test.go"), + c.Assert(strings.Contains(req["filename"].(string), + "kafkahook/kafkahook_test.go"), Equals, true, Commentf(req["filename"].(string))) c.Assert(req["funcName"].(string), Equals, "kafkahook_test.(*KafkaHookTests).TestKafkaHookINFO") @@ -75,8 +75,8 @@ func (s *KafkaHookTests) TestKafkaHookExported(c *C) { c.Assert(req["message"], Equals, "this is a test") c.Assert(req["context"], Equals, nil) c.Assert(req["logLevel"], Equals, "INFO") - c.Assert(strings.HasSuffix(req["filename"].(string), - "github.com/mailgun/logrus-hooks/kafkahook/kafkahook_test.go"), + c.Assert(strings.Contains(req["filename"].(string), + "kafkahook/kafkahook_test.go"), Equals, true, Commentf(req["filename"].(string))) c.Assert(req["funcName"].(string), Equals, "kafkahook_test.(*KafkaHookTests).TestKafkaHookExported") @@ -94,8 +94,8 @@ func (s *KafkaHookTests) TestKafkaHookContext(c *C) { c.Assert(req["message"], Equals, "this is a test") c.Assert(req["lineno"], Equals, float64(91)) c.Assert(req["logLevel"], Equals, "ERROR") - c.Assert(strings.HasSuffix(req["filename"].(string), - "github.com/mailgun/logrus-hooks/kafkahook/kafkahook_test.go"), + c.Assert(strings.Contains(req["filename"].(string), + "kafkahook/kafkahook_test.go"), Equals, true, Commentf(req["filename"].(string))) c.Assert(req["funcName"].(string), Equals, "kafkahook_test.(*KafkaHookTests).TestKafkaHookContext") @@ -156,15 +156,15 @@ func (s *KafkaHookTests) TestFromErr(c *C) { s.log.WithFields(errors.ToLogrus(err)).Info("Info Called") req := GetMsg(s.producer) - c.Assert(strings.HasSuffix(req["filename"].(string), - "github.com/mailgun/logrus-hooks/kafkahook/kafkahook_test.go"), + c.Assert(strings.Contains(req["filename"].(string), + "kafkahook/kafkahook_test.go"), Equals, true, Commentf(req["filename"].(string))) c.Assert(req["lineno"], Equals, float64(154)) c.Assert(req["funcName"], Equals, "kafkahook_test.(*KafkaHookTests).TestFromErr") c.Assert(req["excType"], Equals, "*errors.fundamental") c.Assert(req["excValue"], Equals, "bar: foo") c.Assert(strings.Contains(req["excText"].(string), "(*KafkaHookTests).TestFromErr"), Equals, true) - c.Assert(strings.Contains(req["excText"].(string), "github.com/mailgun/logrus-hooks/kafkahook/kafkahook_test.go"), Equals, true) + c.Assert(strings.Contains(req["excText"].(string), "kafkahook/kafkahook_test.go"), Equals, true) } func GetMsg(producer *mocks.AsyncProducer) map[string]interface{} { diff --git a/levelfilter/levelfilter_test.go b/levelfilter/levelfilter_test.go index c45d8fe..f175eea 100644 --- a/levelfilter/levelfilter_test.go +++ b/levelfilter/levelfilter_test.go @@ -106,8 +106,8 @@ func (s *LevelFilterSuite) TestCallerInfoWithError(c *C) { // Then req := msgGetter() - c.Assert(strings.HasSuffix(req["filename"].(string), - "github.com/mailgun/logrus-hooks/levelfilter/levelfilter_test.go"), + c.Assert(strings.Contains(req["filename"].(string), + "levelfilter/levelfilter_test.go"), Equals, true, Commentf(req["filename"].(string))) c.Assert(req["funcName"], Equals, "levelfilter.(*LevelFilterSuite).TestCallerInfoWithError") @@ -126,8 +126,8 @@ func (s *LevelFilterSuite) TestCallerInfo(c *C) { // Then req := msgGetter() - c.Assert(strings.HasSuffix(req["filename"].(string), - "github.com/mailgun/logrus-hooks/levelfilter/levelfilter_test.go"), Equals, + c.Assert(strings.Contains(req["filename"].(string), + "levelfilter/levelfilter_test.go"), Equals, true, Commentf(req["filename"].(string))) c.Assert(req["funcName"].(string), Equals, "levelfilter.(*LevelFilterSuite).TestCallerInfo") diff --git a/udploghook/udploghook.go b/udploghook/udploghook.go index 1bcf135..08b038e 100644 --- a/udploghook/udploghook.go +++ b/udploghook/udploghook.go @@ -3,29 +3,24 @@ package udploghook import ( "bytes" "fmt" - "io" - "net" - "os" - "path/filepath" - "strings" - - "github.com/mailgun/holster/stack" "github.com/mailgun/logrus-hooks/common" "github.com/mailru/easyjson/jwriter" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "io" + "net" ) type UDPHook struct { - hostName string - appName string + formatter logrus.Formatter conn net.Conn - pid int debug bool } func New(host string, port int) (*UDPHook, error) { - h := UDPHook{} + h := UDPHook{ + formatter: common.DefaultFormatter, + } addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", host, port)) if err != nil { @@ -37,41 +32,16 @@ func New(host string, port int) (*UDPHook, error) { return nil, err } - if h.hostName, err = os.Hostname(); err != nil { - h.hostName = "unknown_host" - } - h.appName = filepath.Base(os.Args[0]) - h.pid = os.Getpid() - return &h, nil } func (h *UDPHook) Fire(entry *logrus.Entry) error { - var caller *stack.FrameInfo - var err error - - caller = common.GetLogrusCaller() - - rec := &common.LogRecord{ - AppName: h.appName, - HostName: h.hostName, - LogLevel: strings.ToUpper(entry.Level.String()), - FileName: caller.File, - FuncName: caller.Func, - LineNo: caller.LineNo, - Message: entry.Message, - Context: nil, - Timestamp: common.Number(float64(entry.Time.UnixNano()) / 1000000000), - PID: h.pid, - } - rec.FromFields(entry.Data) - - // Marshal the log record to JSON string with a category prefix. var w jwriter.Writer w.RawString("logrus:") - rec.MarshalEasyJSON(&w) + w.Raw(h.formatter.Format(entry)) + if w.Error != nil { - return errors.Wrap(w.Error, "UDPHook.Fire() - json.Marshall() error") + return errors.Wrap(w.Error, "while formatting entry") } buf := w.Buffer.BuildBytes() @@ -80,7 +50,7 @@ func (h *UDPHook) Fire(entry *logrus.Entry) error { } // Send the buffer to udplog - err = h.sendUDP(buf) + err := h.sendUDP(buf) if err != nil { return errors.Wrap(err, "UDPHook.Fire()") } diff --git a/udploghook/udploghook_test.go b/udploghook/udploghook_test.go index 5eeda69..dd9cce8 100644 --- a/udploghook/udploghook_test.go +++ b/udploghook/udploghook_test.go @@ -53,8 +53,8 @@ func (s *UDPLogHookTests) TestUDPHookINFO(c *C) { c.Assert(req["message"], Equals, "this is a test") c.Assert(req["context"], Equals, nil) c.Assert(req["logLevel"], Equals, "INFO") - c.Assert(strings.HasSuffix(req["filename"].(string), - "github.com/mailgun/logrus-hooks/udploghook/udploghook_test.go"), + c.Assert(strings.Contains(req["filename"].(string), + "udploghook/udploghook_test.go"), Equals, true, Commentf(req["filename"].(string))) c.Assert(req["funcName"].(string), Equals, "udploghook_test.(*UDPLogHookTests).TestUDPHookINFO") @@ -69,8 +69,8 @@ func (s *UDPLogHookTests) TestUDPHookExported(c *C) { c.Assert(req["message"], Equals, "this is a test") c.Assert(req["context"], Equals, nil) c.Assert(req["logLevel"], Equals, "INFO") - c.Assert(strings.HasSuffix(req["filename"].(string), - "github.com/mailgun/logrus-hooks/udploghook/udploghook_test.go"), + c.Assert(strings.Contains(req["filename"].(string), + "udploghook/udploghook_test.go"), Equals, true, Commentf(req["filename"].(string))) c.Assert(req["funcName"].(string), Equals, "udploghook_test.(*UDPLogHookTests).TestUDPHookExported") @@ -87,8 +87,8 @@ func (s *UDPLogHookTests) TestUDPHookContext(c *C) { req := s.server.GetRequest() c.Assert(req["message"], Equals, "this is a test") c.Assert(req["logLevel"], Equals, "ERROR") - c.Assert(strings.HasSuffix(req["filename"].(string), - "github.com/mailgun/logrus-hooks/udploghook/udploghook_test.go"), + c.Assert(strings.Contains(req["filename"].(string), + "udploghook/udploghook_test.go"), Equals, true, Commentf(req["filename"].(string))) c.Assert(req["funcName"].(string), Equals, "udploghook_test.(*UDPLogHookTests).TestUDPHookContext") @@ -149,8 +149,8 @@ func (s *UDPLogHookTests) TestFromErr(c *C) { s.log.WithFields(errors.ToLogrus(err)).Info("Info Called") req := s.server.GetRequest() - c.Assert(strings.HasSuffix(req["filename"].(string), - "github.com/mailgun/logrus-hooks/udploghook/udploghook_test.go"), + c.Assert(strings.Contains(req["filename"].(string), + "udploghook/udploghook_test.go"), Equals, true, Commentf(req["filename"].(string))) c.Assert(req["lineno"], Equals, float64(147)) c.Assert(req["funcName"].(string), Equals, @@ -158,7 +158,7 @@ func (s *UDPLogHookTests) TestFromErr(c *C) { c.Assert(req["excType"], Equals, "*errors.fundamental") c.Assert(req["excValue"], Equals, "bar: foo") c.Assert(strings.Contains(req["excText"].(string), "(*UDPLogHookTests).TestFromErr"), Equals, true) - c.Assert(strings.Contains(req["excText"].(string), "github.com/mailgun/logrus-hooks/udploghook/udploghook_test.go:147"), Equals, true) + c.Assert(strings.Contains(req["excText"].(string), "udploghook/udploghook_test.go:147"), Equals, true) } func (s *UDPLogHookTests) TestTIDAsString(c *C) {