From e1e2e3502ae92e5e4e6860d5605e4467bafdf510 Mon Sep 17 00:00:00 2001 From: Marvin Blum Date: Mon, 26 Aug 2024 12:07:13 +0200 Subject: [PATCH] Added Caddy IP address header parser, upgraded Go version, updated dependencies. --- CHANGELOG.md | 6 +++ Makefile | 4 +- config/config.toml | 3 +- go.mod | 8 ++-- go.sum | 12 +++--- pkg/proxy/config.go | 6 +++ pkg/proxy/ip.go | 35 +++++++++++++++--- pkg/proxy/ip_test.go | 9 +++++ .../github.com/go-chi/chi/v5/CONTRIBUTING.md | 12 +++--- vendor/github.com/go-chi/chi/v5/context.go | 3 +- .../klauspost/compress/flate/matchlen_amd64.s | 10 ++--- .../klauspost/compress/zstd/dict.go | 31 ++++++++++++++++ .../zstd/internal/xxhash/xxhash_arm64.s | 4 +- .../klauspost/compress/zstd/matchlen_amd64.s | 10 ++--- .../pirsch-go-sdk/v2/pkg/client.go | 25 +++++++++++++ .../pirsch-go-sdk/v2/pkg/types.go | 37 +++++++++++++++++++ vendor/modules.txt | 6 +-- 17 files changed, 177 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47312c6..d069dd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## Changelog +## 2.4.0 + +* added Caddy IP address header parser +* upgraded Go version +* updated dependencies + ## 2.3.0 * removed DNT diff --git a/Makefile b/Makefile index 5b7836f..28405bb 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -.PHONY: test deps docker release +.PHONY: test run deps docker release test: - go test -cover -race github.com/pirsch-analytics/pirsch-go-proxy/pkg/proxy + go test -cover -race ./pkg/... run: go run cmd/main.go config/config.toml diff --git a/config/config.toml b/config/config.toml index ffb566b..249537a 100644 --- a/config/config.toml +++ b/config/config.toml @@ -28,8 +28,9 @@ # This configuration can be used to retreive the real client IP address and set accepted subnets for proxies and load balancers. # Make sure you use the correct header if you are running the proxy behind another proxy or load balancer, otherwise the statistics will be inaccurate. [network] - # Parsed in order, allowed values: CF-Connecting-IP, True-Client-IP, X-Forwarded-For, Forwarded, X-Real-IP + # Parsed in order, allowed values: CF-Connecting-IP, True-Client-IP, X-Forwarded-For, Forwarded, X-Real-IP, caddy # The proxy will use the remote IP if no header is configured. + # For Caddy add "caddy" to the list. It's the same as X-Forwarded-For, but uses the first entry instead of the last. header = ["CF-Connecting-IP", "True-Client-IP", "X-Forwarded-For", "Forwarded", "X-Real-IP"] # List of allowed subnets (CIDR). diff --git a/go.mod b/go.mod index 35423e3..c65ff98 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,14 @@ module github.com/pirsch-analytics/pirsch-go-proxy -go 1.21 +go 1.23 require ( github.com/BurntSushi/toml v1.4.0 github.com/emvi/logbuch v1.2.0 - github.com/go-chi/chi/v5 v5.0.12 + github.com/go-chi/chi/v5 v5.1.0 github.com/go-chi/cors v1.2.1 - github.com/klauspost/compress v1.17.8 - github.com/pirsch-analytics/pirsch-go-sdk/v2 v2.3.0 + github.com/klauspost/compress v1.17.9 + github.com/pirsch-analytics/pirsch-go-sdk/v2 v2.4.0 github.com/stretchr/testify v1.9.0 ) diff --git a/go.sum b/go.sum index 3b6a8d2..487d08a 100644 --- a/go.sum +++ b/go.sum @@ -6,14 +6,14 @@ github.com/emvi/logbuch v1.2.0 h1:Bw0jQH1Dbs+oIygZBNx/2Ub1igXRFtKQrIMRrZdVFJM= github.com/emvi/logbuch v1.2.0/go.mod h1:hFxe0XQOFl76SkE/f0Pt5oQbXRZtyGa8EroBrrbQHuc= github.com/emvi/null v1.3.1 h1:EQxrthMJnzpozGgKrcCepJmwrzA+UCk7ctBWOZLTW0Y= github.com/emvi/null v1.3.1/go.mod h1:tEZhBSLFCucrJdwRentxoOLVRpTCy/3aIfiNKE2uuYU= -github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= -github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= +github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/pirsch-analytics/pirsch-go-sdk/v2 v2.3.0 h1:uZRvFSq3ZL8T0o6HoI0lr2+pl1RpORX7w7Xn+nuUSws= -github.com/pirsch-analytics/pirsch-go-sdk/v2 v2.3.0/go.mod h1:rsaEOobygzywGmIt+fJ4rkL5bJ20ElcBQOXpgOENbcI= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/pirsch-analytics/pirsch-go-sdk/v2 v2.4.0 h1:loPzF29uDuNPptr6KEKt3c+GeHv6GWjUmgqLK+djuz8= +github.com/pirsch-analytics/pirsch-go-sdk/v2 v2.4.0/go.mod h1:rsaEOobygzywGmIt+fJ4rkL5bJ20ElcBQOXpgOENbcI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= diff --git a/pkg/proxy/config.go b/pkg/proxy/config.go index 562cc0a..b1a97ed 100644 --- a/pkg/proxy/config.go +++ b/pkg/proxy/config.go @@ -113,6 +113,12 @@ func loadIPHeader(config *Config) { } } + if strings.ToLower(header) == "caddy" { + ipHeader = append(ipHeader, xForwardedForCaddy) + found = true + break + } + if !found { logbuch.Fatal("Header invalid", logbuch.Fields{"header": header}) } diff --git a/pkg/proxy/ip.go b/pkg/proxy/ip.go index 030a4de..6a75229 100644 --- a/pkg/proxy/ip.go +++ b/pkg/proxy/ip.go @@ -7,12 +7,13 @@ import ( ) var ( - cfConnectingIP = headerParser{"CF-Connecting-IP", parseXForwardedForHeader} - trueClientIP = headerParser{"True-Client-IP", parseXForwardedForHeader} - xForwardedFor = headerParser{"X-Forwarded-For", parseXForwardedForHeader} - forwarded = headerParser{"Forwarded", parseForwardedHeader} - xRealIP = headerParser{"X-Real-IP", parseXRealIPHeader} - allIPHeader = []headerParser{ + cfConnectingIP = headerParser{"CF-Connecting-IP", parseXForwardedForHeader} + trueClientIP = headerParser{"True-Client-IP", parseXForwardedForHeader} + xForwardedFor = headerParser{"X-Forwarded-For", parseXForwardedForHeader} + forwarded = headerParser{"Forwarded", parseForwardedHeader} + xRealIP = headerParser{"X-Real-IP", parseXRealIPHeader} + xForwardedForCaddy = headerParser{"X-Forwarded-For", parseXForwardedForHeader} + allIPHeader = []headerParser{ cfConnectingIP, trueClientIP, xForwardedFor, @@ -119,6 +120,28 @@ func parseXForwardedForHeader(value string) string { return "" } +func parseXForwardedForHeaderFirst(value string) string { + parts := strings.Split(value, ",") + + if len(parts) > 1 { + ip := parts[0] + + if strings.Contains(ip, ":") { + host, _, err := net.SplitHostPort(ip) + + if err != nil { + return ip + } + + return strings.TrimSpace(host) + } + + return strings.TrimSpace(ip) + } + + return "" +} + func parseXRealIPHeader(value string) string { value = cleanIP(strings.TrimSpace(value)) diff --git a/pkg/proxy/ip_test.go b/pkg/proxy/ip_test.go index 1516cdf..7237549 100644 --- a/pkg/proxy/ip_test.go +++ b/pkg/proxy/ip_test.go @@ -59,6 +59,15 @@ func TestParseXForwardedForHeader(t *testing.T) { } } +func TestParseXForwardedForHeaderCaddy(t *testing.T) { + assert.Equal(t, "2003:e1:7f03:8893:5f5e:3681:6c1f:b086", parseXForwardedForHeaderFirst("2003:e1:7f03:8893:5f5e:3681:6c1f:b086, 10.0.1.254")) + assert.Equal(t, "2003:e1:7f03:8893:5f5e:3681:6c1f:b086", parseXForwardedForHeaderFirst("[2003:e1:7f03:8893:5f5e:3681:6c1f:b086]:9876, 10.0.1.254")) + assert.Equal(t, "123.245.221.9", parseXForwardedForHeaderFirst("123.245.221.9:9876, 10.0.1.254")) + assert.Equal(t, "87.171.210.182", parseXForwardedForHeaderFirst("87.171.210.182, 10.0.1.7")) + assert.Empty(t, parseXForwardedForHeaderFirst("2003:e1:7f0c:5700:1e36:e95f:7ef1:401b")) + assert.Empty(t, parseXForwardedForHeaderFirst("123.245.221.9")) +} + func TestParseXRealIPHeader(t *testing.T) { header := []string{ "", diff --git a/vendor/github.com/go-chi/chi/v5/CONTRIBUTING.md b/vendor/github.com/go-chi/chi/v5/CONTRIBUTING.md index c0ac2df..b4a6268 100644 --- a/vendor/github.com/go-chi/chi/v5/CONTRIBUTING.md +++ b/vendor/github.com/go-chi/chi/v5/CONTRIBUTING.md @@ -14,7 +14,7 @@ A typical workflow is: -1. [Fork the repository.][fork] [This tip maybe also helpful.][go-fork-tip] +1. [Fork the repository.][fork] 2. [Create a topic branch.][branch] 3. Add tests for your change. 4. Run `go test`. If your tests pass, return to the step 3. @@ -24,8 +24,8 @@ A typical workflow is: 8. [Submit a pull request.][pull-req] [go-install]: https://golang.org/doc/install -[go-fork-tip]: http://blog.campoy.cat/2014/03/github-and-go-forking-pull-requests-and.html -[fork]: https://help.github.com/articles/fork-a-repo -[branch]: http://learn.github.com/p/branching.html -[git-help]: https://guides.github.com -[pull-req]: https://help.github.com/articles/using-pull-requests +[fork]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo +[branch]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-branches +[git-help]: https://docs.github.com/en +[pull-req]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests + diff --git a/vendor/github.com/go-chi/chi/v5/context.go b/vendor/github.com/go-chi/chi/v5/context.go index 82e5f28..e2cd908 100644 --- a/vendor/github.com/go-chi/chi/v5/context.go +++ b/vendor/github.com/go-chi/chi/v5/context.go @@ -74,9 +74,8 @@ type Context struct { // patterns across a stack of sub-routers. RoutePatterns []string - // methodNotAllowed hint - methodNotAllowed bool methodsAllowed []methodTyp // allowed methods in case of a 405 + methodNotAllowed bool } // Reset a routing context to its initial state. diff --git a/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s b/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s index 9a7655c..0782b86 100644 --- a/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s +++ b/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s @@ -5,7 +5,6 @@ #include "textflag.h" // func matchLen(a []byte, b []byte) int -// Requires: BMI TEXT ·matchLen(SB), NOSPLIT, $0-56 MOVQ a_base+0(FP), AX MOVQ b_base+24(FP), CX @@ -17,17 +16,16 @@ TEXT ·matchLen(SB), NOSPLIT, $0-56 JB matchlen_match4_standalone matchlen_loopback_standalone: - MOVQ (AX)(SI*1), BX - XORQ (CX)(SI*1), BX - TESTQ BX, BX - JZ matchlen_loop_standalone + MOVQ (AX)(SI*1), BX + XORQ (CX)(SI*1), BX + JZ matchlen_loop_standalone #ifdef GOAMD64_v3 TZCNTQ BX, BX #else BSFQ BX, BX #endif - SARQ $0x03, BX + SHRL $0x03, BX LEAL (SI)(BX*1), SI JMP gen_match_len_end diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index 8d5567f..b7b8316 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -273,6 +273,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { enc.Encode(&block, b) addValues(&remain, block.literals) litTotal += len(block.literals) + if len(block.sequences) == 0 { + continue + } seqs += len(block.sequences) block.genCodes() addHist(&ll, block.coders.llEnc.Histogram()) @@ -286,6 +289,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if offset == 0 { continue } + if int(offset) >= len(o.History) { + continue + } if offset > 3 { newOffsets[offset-3]++ } else { @@ -336,6 +342,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if seqs/nUsed < 512 { // Use 512 as minimum. nUsed = seqs / 512 + if nUsed == 0 { + nUsed = 1 + } } copyHist := func(dst *fseEncoder, src *[256]int) ([]byte, error) { hist := dst.Histogram() @@ -358,6 +367,28 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { fakeLength += v hist[i] = uint32(v) } + + // Ensure we aren't trying to represent RLE. + if maxCount == fakeLength { + for i := range hist { + if uint8(i) == maxSym { + fakeLength++ + maxSym++ + hist[i+1] = 1 + if maxSym > 1 { + break + } + } + if hist[0] == 0 { + fakeLength++ + hist[i] = 1 + if maxSym > 1 { + break + } + } + } + } + dst.HistogramFinished(maxSym, maxCount) dst.reUsed = false dst.useRLE = false diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s index 17901e0..ae7d4d3 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s @@ -162,12 +162,12 @@ finalize: MOVD h, ret+24(FP) RET -// func writeBlocks(d *Digest, b []byte) int +// func writeBlocks(s *Digest, b []byte) int TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 LDP ·primes+0(SB), (prime1, prime2) // Load state. Assume v[1-4] are stored contiguously. - MOVD d+0(FP), digest + MOVD s+0(FP), digest LDP 0(digest), (v1, v2) LDP 16(digest), (v3, v4) diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s index 9a7655c..0782b86 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s @@ -5,7 +5,6 @@ #include "textflag.h" // func matchLen(a []byte, b []byte) int -// Requires: BMI TEXT ·matchLen(SB), NOSPLIT, $0-56 MOVQ a_base+0(FP), AX MOVQ b_base+24(FP), CX @@ -17,17 +16,16 @@ TEXT ·matchLen(SB), NOSPLIT, $0-56 JB matchlen_match4_standalone matchlen_loopback_standalone: - MOVQ (AX)(SI*1), BX - XORQ (CX)(SI*1), BX - TESTQ BX, BX - JZ matchlen_loop_standalone + MOVQ (AX)(SI*1), BX + XORQ (CX)(SI*1), BX + JZ matchlen_loop_standalone #ifdef GOAMD64_v3 TZCNTQ BX, BX #else BSFQ BX, BX #endif - SARQ $0x03, BX + SHRL $0x03, BX LEAL (SI)(BX*1), SI JMP gen_match_len_end diff --git a/vendor/github.com/pirsch-analytics/pirsch-go-sdk/v2/pkg/client.go b/vendor/github.com/pirsch-analytics/pirsch-go-sdk/v2/pkg/client.go index 7a571df..c9d8cad 100644 --- a/vendor/github.com/pirsch-analytics/pirsch-go-sdk/v2/pkg/client.go +++ b/vendor/github.com/pirsch-analytics/pirsch-go-sdk/v2/pkg/client.go @@ -58,6 +58,8 @@ const ( tagKeysEndpoint = "/api/v1/statistics/tags" tagDetailsEndpoint = "/api/v1/statistics/tag/details" keywordsEndpoint = "/api/v1/statistics/keywords" + listFunnelEndpoint = "/api/v1/funnel?id=%s" + funnelEndpoint = "/api/v1/statistics/funnel" ) var referrerQueryParams = []string{ @@ -581,6 +583,28 @@ func (client *Client) Keywords(filter *Filter) ([]Keyword, error) { return stats, nil } +// ListFunnel returns a list of all funnels including step definition for given domain ID. +func (client *Client) ListFunnel(id string) ([]Funnel, error) { + funnel := make([]Funnel, 0) + + if err := client.performGet(client.baseURL+fmt.Sprintf(listFunnelEndpoint, id), client.requestRetries, &funnel); err != nil { + return nil, err + } + + return funnel, nil +} + +// Funnel returns a funnel definition and statistics for given funnel ID and filter. +func (client *Client) Funnel(id string, filter *Filter) (*FunnelData, error) { + var funnel FunnelData + + if err := client.performGet(client.getStatsRequestURL(funnelEndpoint, filter)+fmt.Sprintf("&funnel_id=%s", id), client.requestRetries, &funnel); err != nil { + return nil, err + } + + return &funnel, nil +} + func (client *Client) getPageViewData(r *http.Request, options *PageViewOptions) PageView { return PageView{ URL: client.selectField(options.URL, r.URL.String()), @@ -817,6 +841,7 @@ func (client *Client) getStatsRequestURL(endpoint string, filter *Filter) string client.setURLParams(v, "event_meta_key", filter.EventMetaKey) client.setURLParams(v, "language", filter.Language) client.setURLParams(v, "country", filter.Country) + client.setURLParams(v, "region", filter.Region) client.setURLParams(v, "city", filter.City) client.setURLParams(v, "referrer", filter.Referrer) client.setURLParams(v, "referrer_name", filter.ReferrerName) diff --git a/vendor/github.com/pirsch-analytics/pirsch-go-sdk/v2/pkg/types.go b/vendor/github.com/pirsch-analytics/pirsch-go-sdk/v2/pkg/types.go index 3a95073..0ae83a4 100644 --- a/vendor/github.com/pirsch-analytics/pirsch-go-sdk/v2/pkg/types.go +++ b/vendor/github.com/pirsch-analytics/pirsch-go-sdk/v2/pkg/types.go @@ -280,6 +280,7 @@ type EventStats struct { Name string `json:"name"` Visitors int `json:"visitors"` Views int `json:"views"` + Count int `json:"count"` CR float64 `json:"cr"` AverageDurationSeconds int `json:"average_duration_seconds"` MetaKeys []string `json:"meta_keys"` @@ -444,3 +445,39 @@ type Keyword struct { CTR float64 `json:"ctr"` Position float64 `json:"position"` } + +// Funnel is the definition of a funnel. +type Funnel struct { + BaseEntity + + DomainID string `json:"domain_id"` + Name string `json:"name"` + Steps []FunnelStep `json:"steps"` +} + +// FunnelStep is the definition of a funnel step. +type FunnelStep struct { + BaseEntity + + FunnelID string `json:"funnel_id"` + Name string `json:"name"` + Step int `json:"step"` + Filter Filter `json:"filter"` +} + +// FunnelStepData is the result type for a funnel step. +type FunnelStepData struct { + Step int `json:"step"` + Visitors int `json:"visitors"` + RelativeVisitors float64 `json:"relative_visitors"` + PreviousVisitors int `json:"previous_visitors"` + RelativePreviousVisitors float64 `json:"relative_previous_visitors"` + Dropped int `json:"dropped"` + DropOff float64 `json:"drop_off"` +} + +// FunnelData is the response type for the funnel definition and statistics. +type FunnelData struct { + Definition *Funnel `json:"definition"` + Data []FunnelStepData `json:"data"` +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 8ccd741..19dd439 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -11,13 +11,13 @@ github.com/emvi/logbuch # github.com/emvi/null v1.3.1 ## explicit github.com/emvi/null -# github.com/go-chi/chi/v5 v5.0.12 +# github.com/go-chi/chi/v5 v5.1.0 ## explicit; go 1.14 github.com/go-chi/chi/v5 # github.com/go-chi/cors v1.2.1 ## explicit; go 1.14 github.com/go-chi/cors -# github.com/klauspost/compress v1.17.8 +# github.com/klauspost/compress v1.17.9 ## explicit; go 1.20 github.com/klauspost/compress github.com/klauspost/compress/flate @@ -31,7 +31,7 @@ github.com/klauspost/compress/internal/cpuinfo github.com/klauspost/compress/internal/snapref github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd/internal/xxhash -# github.com/pirsch-analytics/pirsch-go-sdk/v2 v2.3.0 +# github.com/pirsch-analytics/pirsch-go-sdk/v2 v2.4.0 ## explicit; go 1.21 github.com/pirsch-analytics/pirsch-go-sdk/v2/pkg # github.com/pmezard/go-difflib v1.0.0