[~] fix nil nil httpRequest (#75)

* [~] fix nil nil httpRequest * [~] fix nil nil httpRequest * chore: Updated coverage badge. --------- Co-authored-by: GitHub Action <[email protected]>
Noooste · Apr 17, 2024 · 5beff80 · 5beff80
1 parent 641d938
commit 5beff80
Show file tree

Hide file tree

Showing 11 changed files with 65 additions and 186 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # AzureTLS Client 
 [![GoDoc](https://godoc.org/github.com/Noooste/azuretls-client?status.svg)](https://godoc.org/github.com/Noooste/azuretls-client)
-![Coverage](https://img.shields.io/badge/Coverage-80.8%25-brightgreen)
+![Coverage](https://img.shields.io/badge/Coverage-80.2%25-brightgreen)
 [![build](https://github.com/Noooste/azuretls-client/actions/workflows/push.yml/badge.svg)](https://github.com/Noooste/azuretls-client/actions/workflows/push.yml)
 [![Go Report Card](https://goreportcard.com/badge/Noooste/azuretls-client)](https://goreportcard.com/report/Noooste/azuretls-client)
 [![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/Noooste/azuretls-client/blob/master/LICENSE)

diff --git a/connection.go b/connection.go
@@ -371,17 +371,22 @@ func (c *Conn) NewTLS(addr string) (err error) {
 		ServerName:         hostname,
 		InsecureSkipVerify: c.InsecureSkipVerify,
 		VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
-			if c.PinManager == nil {
-				return nil
+			if c.PinManager != nil {
+				for _, chain := range verifiedChains {
+					for _, cert := range chain {
+						if c.PinManager.Verify(cert) {
+							return nil
+						}
+					}
+				}
+
+				return errors.New("pin verification failed")
 			}
 
 			now := time.Now()
+
 			for _, chain := range verifiedChains {
 				for _, cert := range chain {
-					if c.PinManager.Verify(cert) {
-						return nil
-					}
-
 					if now.Before(cert.NotBefore) {
 						return errors.New("certificate is not valid yet")
 					}
@@ -400,7 +405,7 @@ func (c *Conn) NewTLS(addr string) (err error) {
 				}
 			}
 
-			return errors.New("pin verification failed")
+			return nil
 		},
 	}
 

diff --git a/go.mod b/go.mod
@@ -3,23 +3,23 @@ module github.com/Noooste/azuretls-client
 go 1.20
 
 require (
-	github.com/Noooste/fhttp v1.0.11
-	github.com/Noooste/utls v1.2.7
+	github.com/Noooste/fhttp v1.0.12
+	github.com/Noooste/utls v1.2.9
 	github.com/Noooste/websocket v1.0.3
 	github.com/fatih/color v1.16.0
-	golang.org/x/net v0.23.0
+	golang.org/x/net v0.24.0
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
-	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/quic-go/quic-go v0.42.0 // indirect
-	github.com/refraction-networking/utls v1.6.3 // indirect
+	github.com/refraction-networking/utls v1.6.4 // indirect
 	golang.org/x/crypto v0.22.0 // indirect
-	golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
+	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 )
diff --git a/go.sum b/go.sum
@@ -2,8 +2,14 @@ github.com/Noooste/fhttp v1.0.8 h1:iLSM75L7SInEirfdvwJUrUd/Y3AeF1LwpMuOQMM0zEg=
 github.com/Noooste/fhttp v1.0.8/go.mod h1:CMVxKOhNheqJN5HYE4Rlvz2SRdV8Uv7YWmi6OwmB/Bk=
 github.com/Noooste/fhttp v1.0.11 h1:tpVXalX6gCkNaUA694RJ/bIUh2Ypzg8vXKJMnS9mpDM=
 github.com/Noooste/fhttp v1.0.11/go.mod h1:CMVxKOhNheqJN5HYE4Rlvz2SRdV8Uv7YWmi6OwmB/Bk=
+github.com/Noooste/fhttp v1.0.12 h1:2N15bIATKaC6q+LVyRGyxPyuqEPvwAS3Uk1peC3YVHU=
+github.com/Noooste/fhttp v1.0.12/go.mod h1:CMVxKOhNheqJN5HYE4Rlvz2SRdV8Uv7YWmi6OwmB/Bk=
 github.com/Noooste/utls v1.2.7 h1:NLlRybZDzW+dXk/Uavb2E+pWeK+GAH2XkWq3g+C/WA0=
 github.com/Noooste/utls v1.2.7/go.mod h1:MRUEmRiDO6ORKziZ2ObNwMjxy0vRviJ91JF1qVa0loM=
+github.com/Noooste/utls v1.2.8 h1:/sv+9cDpNMKyyAzEyThptQUsOPV0kmzapET63BDEBHI=
+github.com/Noooste/utls v1.2.8/go.mod h1:dHM5MlRyB/ieog9axOxkEh4qZRgw5xA406y7sEdVmoQ=
+github.com/Noooste/utls v1.2.9 h1:VLNs0WmPFjswU4PxXTHl0AY4sarHi+638c3cSI9Hsng=
+github.com/Noooste/utls v1.2.9/go.mod h1:dHM5MlRyB/ieog9axOxkEh4qZRgw5xA406y7sEdVmoQ=
 github.com/Noooste/websocket v1.0.3 h1:drW7tvZ3YqzqI9wApnaH1Q0syFMXO7gbLlsBWjZvMNA=
 github.com/Noooste/websocket v1.0.3/go.mod h1:Qhw0Rtuju/fPPbcb3R5XGq7poa51qPDL462jTltl9nQ=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
@@ -18,6 +24,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
 github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
 github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -31,16 +39,22 @@ github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utp
 github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
 github.com/refraction-networking/utls v1.6.3 h1:MFOfRN35sSx6K5AZNIoESsBuBxS2LCgRilRIdHb6fDc=
 github.com/refraction-networking/utls v1.6.3/go.mod h1:yil9+7qSl+gBwJqztoQseO6Pr3h62pQoY1lXiNR/FPs=
+github.com/refraction-networking/utls v1.6.4 h1:aeynTroaYn7y+mFtqv8D0bQ4bw0y9nJHneGxJ7lvRDM=
+github.com/refraction-networking/utls v1.6.4/go.mod h1:2VL2xfiqgFAZtJKeUTlf+PSYFs3Eu7km0gCtXJ3m8zs=
 golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
 golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
+golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
+golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
 golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
 golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=

diff --git a/header.go b/header.go
@@ -192,7 +192,7 @@ func (s *Session) makeHeadersCopier(ireq *Request) func(*Request) {
 		icookies map[string][]*http.Cookie
 	)
 
-	if s.CookieJar != nil && ireq.Header.Get("Cookie") != "" {
+	if s.CookieJar != nil && ireq.HttpRequest != nil && ireq.Header.Get("Cookie") != "" {
 		icookies = make(map[string][]*http.Cookie)
 		for _, c := range ireq.HttpRequest.Cookies() {
 			icookies[c.Name] = append(icookies[c.Name], c)

diff --git a/request.go b/request.go
@@ -101,7 +101,9 @@ func (s *Session) fillEmptyValues(request *Request) {
 		request.MaxRedirects = s.MaxRedirects
 	}
 
-	request.InsecureSkipVerify = s.InsecureSkipVerify
+	if !request.InsecureSkipVerify {
+		request.InsecureSkipVerify = s.InsecureSkipVerify
+	}
 }
 
 func (s *Session) buildRequest(ctx context.Context, req *Request) (err error) {

diff --git a/session.go b/session.go
@@ -257,9 +257,27 @@ func (s *Session) do(req *Request, args ...any) (resp *Response, err error) {
 	}
 
 	var cancel context.CancelFunc
-	req.ctx, cancel = context.WithDeadline(req.ctx, req.deadline)
 
-	defer cancel()
+	if !req.IgnoreBody {
+		req.ctx, cancel = context.WithDeadline(req.ctx, req.deadline)
+	}
+
+	defer func() {
+		if cancel != nil {
+			cancel()
+		}
+	}()
+
+	if req.DisableRedirects {
+		req.startTime = time.Now()
+		resp, err = s.send(req)
+		if err != nil {
+			return
+		}
+		req.CloseBody()
+		req.Response = resp
+		return
+	}
 
 	var reqs = make([]*Request, 0, req.MaxRedirects+1)
 
@@ -274,6 +292,10 @@ func (s *Session) do(req *Request, args ...any) (resp *Response, err error) {
 		// For all but the first request, create the next
 		// request hop and replace req.
 		if len(reqs) > 0 {
+			if resp == nil {
+				return nil, errors.New("internal error: nil response")
+			}
+
 			loc := resp.Header.Get("Location")
 			if loc == "" {
 				_ = resp.CloseBody()
@@ -339,11 +361,6 @@ func (s *Session) do(req *Request, args ...any) (resp *Response, err error) {
 
 		req.Response = resp
 
-		if req.DisableRedirects {
-			req.CloseBody()
-			return resp, nil
-		}
-
 		var shouldRedirect bool
 
 		redirectMethod, shouldRedirect, includeBody = RedirectBehavior(req.Method, resp, reqs[0])

diff --git a/structs.go b/structs.go
@@ -148,6 +148,8 @@ type Request struct {
 	InsecureSkipVerify bool
 
 	// If true, the body of the response is not read.
+	// The response body can be read from Response.RawBody and
+	// you will have to close using Response.CloseBody manually.
 	IgnoreBody bool
 	Proto      string
 	ForceHTTP1 bool

diff --git a/test/header_test.go b/test/header_test.go
@@ -147,132 +147,6 @@ func checkOrder(response scrapeResponse, expectedOrder []string) bool {
 	return true
 }
 
-func TestHeaderWithinSession(t *testing.T) {
-	session := azuretls.NewSession()
-	defer session.Close()
-
-	session.HeaderOrder = []string{"cache-control", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform", "upgrade-insecure-requests", "user-agent", "accept", "sec-fetch-site", "sec-fetch-mode", "sec-fetch-user", "sec-fetch-dest", "accept-encoding", "accept-language"}
-	session.Header = http.Header{
-		"accept":                    {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"},
-		"accept-encoding":           {"gzip, deflate, br"},
-		"accept-language":           {"zh-HK,zh-TW;q=0.9,zh;q=0.8"},
-		"cache-control":             {"max-age=0"},
-		"sec-ch-ua":                 {`"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"`},
-		"sec-ch-ua-mobile":          {"?0"},
-		"sec-ch-ua-platform":        {`"Windows"`},
-		"sec-fetch-site":            {"none"},
-		"sec-fetch-mode":            {"navigate"},
-		"sec-fetch-user":            {"?1"},
-		"sec-fetch-dest":            {"document"},
-		"user-agent":                {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"},
-		"upgrade-insecure-requests": {"1"},
-	}
-
-	response, err := session.Get("https://tools.scrapfly.io/api/fp/anything")
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if response.StatusCode != 200 {
-		t.Fatal("TestHeader failed, expected: 200, got: ", response.StatusCode)
-	}
-
-	var parsedResponse scrapeResponse
-
-	if err = response.JSON(&parsedResponse); err != nil {
-		t.Fatal(err)
-	}
-
-	if !checkOrder(parsedResponse, session.HeaderOrder) {
-		t.Fatal("TestHeader failed, headers order is not as expected")
-	}
-}
-
-func TestHeaderInParam(t *testing.T) {
-	session := azuretls.NewSession()
-	defer session.Close()
-
-	session.HeaderOrder = []string{"cache-control", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform", "upgrade-insecure-requests", "user-agent", "accept", "sec-fetch-site", "sec-fetch-mode", "sec-fetch-user", "sec-fetch-dest", "accept-encoding", "accept-language"}
-	headers := http.Header{
-		"accept":                    {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"},
-		"accept-encoding":           {"gzip, deflate, br"},
-		"accept-language":           {"zh-HK,zh-TW;q=0.9,zh;q=0.8"},
-		"cache-control":             {"max-age=0"},
-		"sec-ch-ua":                 {`"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"`},
-		"sec-ch-ua-mobile":          {"?0"},
-		"sec-ch-ua-platform":        {`"Windows"`},
-		"sec-fetch-site":            {"none"},
-		"sec-fetch-mode":            {"navigate"},
-		"sec-fetch-user":            {"?1"},
-		"sec-fetch-dest":            {"document"},
-		"user-agent":                {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"},
-		"upgrade-insecure-requests": {"1"},
-	}
-
-	response, err := session.Get("https://tools.scrapfly.io/api/fp/anything", headers)
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if response.StatusCode != 200 {
-		t.Fatal("TestHeader failed, expected: 200, got: ", response.StatusCode)
-	}
-
-	var parsedResponse scrapeResponse
-
-	if err = response.JSON(&parsedResponse); err != nil {
-		t.Fatal(err)
-	}
-
-	if !checkOrder(parsedResponse, session.HeaderOrder) {
-		t.Fatal("TestHeader failed, headers order is not as expected")
-	}
-}
-
-func TestHeaderAndOrderInParam(t *testing.T) {
-	session := azuretls.NewSession()
-	defer session.Close()
-
-	order := azuretls.HeaderOrder{"cache-control", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform", "upgrade-insecure-requests", "user-agent", "accept", "sec-fetch-site", "sec-fetch-mode", "sec-fetch-user", "sec-fetch-dest", "accept-encoding", "accept-language"}
-	headers := http.Header{
-		"accept":                    {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"},
-		"accept-encoding":           {"gzip, deflate, br"},
-		"accept-language":           {"zh-HK,zh-TW;q=0.9,zh;q=0.8"},
-		"cache-control":             {"max-age=0"},
-		"sec-ch-ua":                 {`"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"`},
-		"sec-ch-ua-mobile":          {"?0"},
-		"sec-ch-ua-platform":        {`"Windows"`},
-		"sec-fetch-site":            {"none"},
-		"sec-fetch-mode":            {"navigate"},
-		"sec-fetch-user":            {"?1"},
-		"sec-fetch-dest":            {"document"},
-		"user-agent":                {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"},
-		"upgrade-insecure-requests": {"1"},
-	}
-
-	response, err := session.Get("https://tools.scrapfly.io/api/fp/anything", headers, order)
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if response.StatusCode != 200 {
-		t.Fatal("TestHeader failed, expected: 200, got: ", response.StatusCode)
-	}
-
-	var parsedResponse scrapeResponse
-
-	if err = response.JSON(&parsedResponse); err != nil {
-		t.Fatal(err)
-	}
-
-	if !checkOrder(parsedResponse, order) {
-		t.Fatal("TestHeader failed, headers order is not as expected")
-	}
-}
-
 func TestHeaderAndOrderInParamWithRedirect(t *testing.T) {
 	session := azuretls.NewSession()
 	defer session.Close()

diff --git a/test/response_test.go b/test/response_test.go
@@ -11,7 +11,7 @@ func TestResponse_CloseBody(t *testing.T) {
 	session := azuretls.NewSession()
 
 	response, err := session.Do(&azuretls.Request{
-		Method:     "GET",
+		Method:     http.MethodGet,
 		Url:        "https://tls.peet.ws/api/all",
 		IgnoreBody: true,
 	})
@@ -21,7 +21,7 @@ func TestResponse_CloseBody(t *testing.T) {
 	}
 
 	if _, err = io.ReadAll(response.RawBody); err != nil {
-		t.Fatal("TestResponse_CloseBody failed, expected: nil, got: ", err)
+		t.Fatal("TestResponse_ReadAll failed, expected: nil, got: ", err)
 	}
 
 	if err = response.CloseBody(); err != nil {

diff --git a/test/session_test.go b/test/session_test.go
@@ -591,41 +591,6 @@ func TestSession_SetContext(t *testing.T) {
 	}
 }
 
-func TestSession_Context(t *testing.T) {
-	stop := make(chan bool)
-
-	for i := 0; i <= 5; i++ {
-		go func() {
-			ctx, cancel := context.WithCancel(context.Background())
-
-			go func() {
-				<-stop
-				cancel()
-			}()
-
-			session := azuretls.NewSessionWithContext(ctx)
-			defer session.Close()
-
-			if err := session.SetProxy(os.Getenv("NON_SECURE_PROXY")); err != nil {
-				fmt.Println(err)
-				return
-			}
-
-			response, err := session.Get("https://testfile.org/files-5GB")
-
-			if err != nil {
-				fmt.Println(err)
-			} else {
-				fmt.Println(response.StatusCode, string(response.Body))
-			}
-		}()
-	}
-
-	time.Sleep(500 * time.Millisecond)
-	stop <- true
-	time.Sleep(1 * time.Second)
-}
-
 func TestSession_ContextError(t *testing.T) {
 	exampleContext, ca := context.WithCancel(context.Background())
 	defer ca()