From 5beff8003af255df1b6f54ecbeaf35ddcb896cfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nos=D1=82e?= <83548733+Noooste@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:45:50 +0200 Subject: [PATCH] [~] fix nil nil httpRequest (#75) * [~] fix nil nil httpRequest * [~] fix nil nil httpRequest * chore: Updated coverage badge. --------- Co-authored-by: GitHub Action --- README.md | 2 +- connection.go | 19 ++++--- go.mod | 12 ++-- go.sum | 14 +++++ header.go | 2 +- request.go | 4 +- session.go | 31 ++++++++--- structs.go | 2 + test/header_test.go | 126 ------------------------------------------ test/response_test.go | 4 +- test/session_test.go | 35 ------------ 11 files changed, 65 insertions(+), 186 deletions(-) diff --git a/README.md b/README.md index 85f6452..46b7925 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # AzureTLS Client [![GoDoc](https://godoc.org/github.com/Noooste/azuretls-client?status.svg)](https://godoc.org/github.com/Noooste/azuretls-client) -![Coverage](https://img.shields.io/badge/Coverage-80.8%25-brightgreen) +![Coverage](https://img.shields.io/badge/Coverage-80.2%25-brightgreen) [![build](https://github.com/Noooste/azuretls-client/actions/workflows/push.yml/badge.svg)](https://github.com/Noooste/azuretls-client/actions/workflows/push.yml) [![Go Report Card](https://goreportcard.com/badge/Noooste/azuretls-client)](https://goreportcard.com/report/Noooste/azuretls-client) [![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/Noooste/azuretls-client/blob/master/LICENSE) diff --git a/connection.go b/connection.go index dae8ae1..b492f74 100644 --- a/connection.go +++ b/connection.go @@ -371,17 +371,22 @@ func (c *Conn) NewTLS(addr string) (err error) { ServerName: hostname, InsecureSkipVerify: c.InsecureSkipVerify, VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { - if c.PinManager == nil { - return nil + if c.PinManager != nil { + for _, chain := range verifiedChains { + for _, cert := range chain { + if c.PinManager.Verify(cert) { + return nil + } + } + } + + return errors.New("pin verification failed") } now := time.Now() + for _, chain := range verifiedChains { for _, cert := range chain { - if c.PinManager.Verify(cert) { - return nil - } - if now.Before(cert.NotBefore) { return errors.New("certificate is not valid yet") } @@ -400,7 +405,7 @@ func (c *Conn) NewTLS(addr string) (err error) { } } - return errors.New("pin verification failed") + return nil }, } diff --git a/go.mod b/go.mod index d55cc88..7cd9078 100644 --- a/go.mod +++ b/go.mod @@ -3,23 +3,23 @@ module github.com/Noooste/azuretls-client go 1.20 require ( - github.com/Noooste/fhttp v1.0.11 - github.com/Noooste/utls v1.2.7 + github.com/Noooste/fhttp v1.0.12 + github.com/Noooste/utls v1.2.9 github.com/Noooste/websocket v1.0.3 github.com/fatih/color v1.16.0 - golang.org/x/net v0.23.0 + golang.org/x/net v0.24.0 ) require ( github.com/andybalholm/brotli v1.1.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/quic-go/quic-go v0.42.0 // indirect - github.com/refraction-networking/utls v1.6.3 // indirect + github.com/refraction-networking/utls v1.6.4 // indirect golang.org/x/crypto v0.22.0 // indirect - golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect ) diff --git a/go.sum b/go.sum index 877187c..0595eff 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,14 @@ github.com/Noooste/fhttp v1.0.8 h1:iLSM75L7SInEirfdvwJUrUd/Y3AeF1LwpMuOQMM0zEg= github.com/Noooste/fhttp v1.0.8/go.mod h1:CMVxKOhNheqJN5HYE4Rlvz2SRdV8Uv7YWmi6OwmB/Bk= github.com/Noooste/fhttp v1.0.11 h1:tpVXalX6gCkNaUA694RJ/bIUh2Ypzg8vXKJMnS9mpDM= github.com/Noooste/fhttp v1.0.11/go.mod h1:CMVxKOhNheqJN5HYE4Rlvz2SRdV8Uv7YWmi6OwmB/Bk= +github.com/Noooste/fhttp v1.0.12 h1:2N15bIATKaC6q+LVyRGyxPyuqEPvwAS3Uk1peC3YVHU= +github.com/Noooste/fhttp v1.0.12/go.mod h1:CMVxKOhNheqJN5HYE4Rlvz2SRdV8Uv7YWmi6OwmB/Bk= github.com/Noooste/utls v1.2.7 h1:NLlRybZDzW+dXk/Uavb2E+pWeK+GAH2XkWq3g+C/WA0= github.com/Noooste/utls v1.2.7/go.mod h1:MRUEmRiDO6ORKziZ2ObNwMjxy0vRviJ91JF1qVa0loM= +github.com/Noooste/utls v1.2.8 h1:/sv+9cDpNMKyyAzEyThptQUsOPV0kmzapET63BDEBHI= +github.com/Noooste/utls v1.2.8/go.mod h1:dHM5MlRyB/ieog9axOxkEh4qZRgw5xA406y7sEdVmoQ= +github.com/Noooste/utls v1.2.9 h1:VLNs0WmPFjswU4PxXTHl0AY4sarHi+638c3cSI9Hsng= +github.com/Noooste/utls v1.2.9/go.mod h1:dHM5MlRyB/ieog9axOxkEh4qZRgw5xA406y7sEdVmoQ= github.com/Noooste/websocket v1.0.3 h1:drW7tvZ3YqzqI9wApnaH1Q0syFMXO7gbLlsBWjZvMNA= github.com/Noooste/websocket v1.0.3/go.mod h1:Qhw0Rtuju/fPPbcb3R5XGq7poa51qPDL462jTltl9nQ= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= @@ -18,6 +24,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -31,16 +39,22 @@ github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utp github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.6.3 h1:MFOfRN35sSx6K5AZNIoESsBuBxS2LCgRilRIdHb6fDc= github.com/refraction-networking/utls v1.6.3/go.mod h1:yil9+7qSl+gBwJqztoQseO6Pr3h62pQoY1lXiNR/FPs= +github.com/refraction-networking/utls v1.6.4 h1:aeynTroaYn7y+mFtqv8D0bQ4bw0y9nJHneGxJ7lvRDM= +github.com/refraction-networking/utls v1.6.4/go.mod h1:2VL2xfiqgFAZtJKeUTlf+PSYFs3Eu7km0gCtXJ3m8zs= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= diff --git a/header.go b/header.go index a1b1fb8..c6003c0 100644 --- a/header.go +++ b/header.go @@ -192,7 +192,7 @@ func (s *Session) makeHeadersCopier(ireq *Request) func(*Request) { icookies map[string][]*http.Cookie ) - if s.CookieJar != nil && ireq.Header.Get("Cookie") != "" { + if s.CookieJar != nil && ireq.HttpRequest != nil && ireq.Header.Get("Cookie") != "" { icookies = make(map[string][]*http.Cookie) for _, c := range ireq.HttpRequest.Cookies() { icookies[c.Name] = append(icookies[c.Name], c) diff --git a/request.go b/request.go index 6cf450e..96c7547 100644 --- a/request.go +++ b/request.go @@ -101,7 +101,9 @@ func (s *Session) fillEmptyValues(request *Request) { request.MaxRedirects = s.MaxRedirects } - request.InsecureSkipVerify = s.InsecureSkipVerify + if !request.InsecureSkipVerify { + request.InsecureSkipVerify = s.InsecureSkipVerify + } } func (s *Session) buildRequest(ctx context.Context, req *Request) (err error) { diff --git a/session.go b/session.go index f0ca1f4..6b809f9 100644 --- a/session.go +++ b/session.go @@ -257,9 +257,27 @@ func (s *Session) do(req *Request, args ...any) (resp *Response, err error) { } var cancel context.CancelFunc - req.ctx, cancel = context.WithDeadline(req.ctx, req.deadline) - defer cancel() + if !req.IgnoreBody { + req.ctx, cancel = context.WithDeadline(req.ctx, req.deadline) + } + + defer func() { + if cancel != nil { + cancel() + } + }() + + if req.DisableRedirects { + req.startTime = time.Now() + resp, err = s.send(req) + if err != nil { + return + } + req.CloseBody() + req.Response = resp + return + } var reqs = make([]*Request, 0, req.MaxRedirects+1) @@ -274,6 +292,10 @@ func (s *Session) do(req *Request, args ...any) (resp *Response, err error) { // For all but the first request, create the next // request hop and replace req. if len(reqs) > 0 { + if resp == nil { + return nil, errors.New("internal error: nil response") + } + loc := resp.Header.Get("Location") if loc == "" { _ = resp.CloseBody() @@ -339,11 +361,6 @@ func (s *Session) do(req *Request, args ...any) (resp *Response, err error) { req.Response = resp - if req.DisableRedirects { - req.CloseBody() - return resp, nil - } - var shouldRedirect bool redirectMethod, shouldRedirect, includeBody = RedirectBehavior(req.Method, resp, reqs[0]) diff --git a/structs.go b/structs.go index f104e57..cf1b171 100644 --- a/structs.go +++ b/structs.go @@ -148,6 +148,8 @@ type Request struct { InsecureSkipVerify bool // If true, the body of the response is not read. + // The response body can be read from Response.RawBody and + // you will have to close using Response.CloseBody manually. IgnoreBody bool Proto string ForceHTTP1 bool diff --git a/test/header_test.go b/test/header_test.go index f92391e..2008cb0 100644 --- a/test/header_test.go +++ b/test/header_test.go @@ -147,132 +147,6 @@ func checkOrder(response scrapeResponse, expectedOrder []string) bool { return true } -func TestHeaderWithinSession(t *testing.T) { - session := azuretls.NewSession() - defer session.Close() - - session.HeaderOrder = []string{"cache-control", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform", "upgrade-insecure-requests", "user-agent", "accept", "sec-fetch-site", "sec-fetch-mode", "sec-fetch-user", "sec-fetch-dest", "accept-encoding", "accept-language"} - session.Header = http.Header{ - "accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"}, - "accept-encoding": {"gzip, deflate, br"}, - "accept-language": {"zh-HK,zh-TW;q=0.9,zh;q=0.8"}, - "cache-control": {"max-age=0"}, - "sec-ch-ua": {`"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"`}, - "sec-ch-ua-mobile": {"?0"}, - "sec-ch-ua-platform": {`"Windows"`}, - "sec-fetch-site": {"none"}, - "sec-fetch-mode": {"navigate"}, - "sec-fetch-user": {"?1"}, - "sec-fetch-dest": {"document"}, - "user-agent": {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}, - "upgrade-insecure-requests": {"1"}, - } - - response, err := session.Get("https://tools.scrapfly.io/api/fp/anything") - - if err != nil { - t.Fatal(err) - } - - if response.StatusCode != 200 { - t.Fatal("TestHeader failed, expected: 200, got: ", response.StatusCode) - } - - var parsedResponse scrapeResponse - - if err = response.JSON(&parsedResponse); err != nil { - t.Fatal(err) - } - - if !checkOrder(parsedResponse, session.HeaderOrder) { - t.Fatal("TestHeader failed, headers order is not as expected") - } -} - -func TestHeaderInParam(t *testing.T) { - session := azuretls.NewSession() - defer session.Close() - - session.HeaderOrder = []string{"cache-control", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform", "upgrade-insecure-requests", "user-agent", "accept", "sec-fetch-site", "sec-fetch-mode", "sec-fetch-user", "sec-fetch-dest", "accept-encoding", "accept-language"} - headers := http.Header{ - "accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"}, - "accept-encoding": {"gzip, deflate, br"}, - "accept-language": {"zh-HK,zh-TW;q=0.9,zh;q=0.8"}, - "cache-control": {"max-age=0"}, - "sec-ch-ua": {`"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"`}, - "sec-ch-ua-mobile": {"?0"}, - "sec-ch-ua-platform": {`"Windows"`}, - "sec-fetch-site": {"none"}, - "sec-fetch-mode": {"navigate"}, - "sec-fetch-user": {"?1"}, - "sec-fetch-dest": {"document"}, - "user-agent": {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}, - "upgrade-insecure-requests": {"1"}, - } - - response, err := session.Get("https://tools.scrapfly.io/api/fp/anything", headers) - - if err != nil { - t.Fatal(err) - } - - if response.StatusCode != 200 { - t.Fatal("TestHeader failed, expected: 200, got: ", response.StatusCode) - } - - var parsedResponse scrapeResponse - - if err = response.JSON(&parsedResponse); err != nil { - t.Fatal(err) - } - - if !checkOrder(parsedResponse, session.HeaderOrder) { - t.Fatal("TestHeader failed, headers order is not as expected") - } -} - -func TestHeaderAndOrderInParam(t *testing.T) { - session := azuretls.NewSession() - defer session.Close() - - order := azuretls.HeaderOrder{"cache-control", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform", "upgrade-insecure-requests", "user-agent", "accept", "sec-fetch-site", "sec-fetch-mode", "sec-fetch-user", "sec-fetch-dest", "accept-encoding", "accept-language"} - headers := http.Header{ - "accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"}, - "accept-encoding": {"gzip, deflate, br"}, - "accept-language": {"zh-HK,zh-TW;q=0.9,zh;q=0.8"}, - "cache-control": {"max-age=0"}, - "sec-ch-ua": {`"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"`}, - "sec-ch-ua-mobile": {"?0"}, - "sec-ch-ua-platform": {`"Windows"`}, - "sec-fetch-site": {"none"}, - "sec-fetch-mode": {"navigate"}, - "sec-fetch-user": {"?1"}, - "sec-fetch-dest": {"document"}, - "user-agent": {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}, - "upgrade-insecure-requests": {"1"}, - } - - response, err := session.Get("https://tools.scrapfly.io/api/fp/anything", headers, order) - - if err != nil { - t.Fatal(err) - } - - if response.StatusCode != 200 { - t.Fatal("TestHeader failed, expected: 200, got: ", response.StatusCode) - } - - var parsedResponse scrapeResponse - - if err = response.JSON(&parsedResponse); err != nil { - t.Fatal(err) - } - - if !checkOrder(parsedResponse, order) { - t.Fatal("TestHeader failed, headers order is not as expected") - } -} - func TestHeaderAndOrderInParamWithRedirect(t *testing.T) { session := azuretls.NewSession() defer session.Close() diff --git a/test/response_test.go b/test/response_test.go index de74a9f..2103cd0 100644 --- a/test/response_test.go +++ b/test/response_test.go @@ -11,7 +11,7 @@ func TestResponse_CloseBody(t *testing.T) { session := azuretls.NewSession() response, err := session.Do(&azuretls.Request{ - Method: "GET", + Method: http.MethodGet, Url: "https://tls.peet.ws/api/all", IgnoreBody: true, }) @@ -21,7 +21,7 @@ func TestResponse_CloseBody(t *testing.T) { } if _, err = io.ReadAll(response.RawBody); err != nil { - t.Fatal("TestResponse_CloseBody failed, expected: nil, got: ", err) + t.Fatal("TestResponse_ReadAll failed, expected: nil, got: ", err) } if err = response.CloseBody(); err != nil { diff --git a/test/session_test.go b/test/session_test.go index 17e4f68..c837f3a 100644 --- a/test/session_test.go +++ b/test/session_test.go @@ -591,41 +591,6 @@ func TestSession_SetContext(t *testing.T) { } } -func TestSession_Context(t *testing.T) { - stop := make(chan bool) - - for i := 0; i <= 5; i++ { - go func() { - ctx, cancel := context.WithCancel(context.Background()) - - go func() { - <-stop - cancel() - }() - - session := azuretls.NewSessionWithContext(ctx) - defer session.Close() - - if err := session.SetProxy(os.Getenv("NON_SECURE_PROXY")); err != nil { - fmt.Println(err) - return - } - - response, err := session.Get("https://testfile.org/files-5GB") - - if err != nil { - fmt.Println(err) - } else { - fmt.Println(response.StatusCode, string(response.Body)) - } - }() - } - - time.Sleep(500 * time.Millisecond) - stop <- true - time.Sleep(1 * time.Second) -} - func TestSession_ContextError(t *testing.T) { exampleContext, ca := context.WithCancel(context.Background()) defer ca()