diff --git a/.gitignore b/.gitignore index f398beae7..4963ee75b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ output /**/*.log profile.out coverage.txt +.idea/* +.vscode/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a58d5ed5..bd66ae1e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,29 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [v1.1.0] - 2021-04-08 + +### Added +- Support JA3 fingerprint for SSL/TLS client +- Support Slow‑Start to allow a backend instance gradually recover its weight +- Add maxConnPerHost to limit the number of connections to a backend +- mod_header: add header renaming actions +- Merge some updates from golang/net/textproto +- Merge some updates from golang/net/http +- Merge some updates from golang/net/http2 +- Documents optimization + +### Changed +- Change outlierDetectionLevel to OutlierDetectionHttpCode + +### Fixed +- Fix panic when write internal response timeout +- Fix unit test in bfe_spdy/frame_test.go under go 1.16 + +### Security +- Fix config loading for multi-value option + + ## [v1.0.0] - 2021-01-15 ### Added @@ -215,7 +238,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Flexible plugin framework to extend functionality. Based on the framework, developer can add new features rapidly - Detailed built-in metrics available for service status monitor - +[v1.1.0]: https://github.com/bfenetworks/bfe/compare/v1.0.0...v1.1.0 [v1.0.0]: https://github.com/bfenetworks/bfe/compare/v0.12.0...v1.0.0 [v0.12.0]: https://github.com/bfenetworks/bfe/compare/v0.11.0...v0.12.0 [v0.11.0]: https://github.com/bfenetworks/bfe/compare/v0.10.0...v0.11.0 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 012d7c12d..3327464a6 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -16,6 +16,7 @@ | Hao Dong | anotherwriter | | Haobin zhang | zhanghaobin | | Hui Yu | dblate | +| Guangze Song | coopersong | | Gen Wang | gracewang510 | | Jie Liu | freeHackOfJeff | | Jie Wan | wanjiecs | @@ -41,6 +42,7 @@ | Shan Xiao | arlingtonroad | | Shengnan Yu | goldfish-fish | | Shuai Yan | yanshuai615270 | +| Shuo Yang | yangshuothtf | | Sijie Yang | iyangsj | | Tianqi Zhang | NKztq | | Weijie Zhao | zwj13513118235 | diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 3d71e318c..cadcf4def 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -18,5 +18,6 @@ This file lists who are the maintainers of the BFE project. The responsibilities | ---- | --------- | ----------- | | [Derek Zheng](mailto:shanhu5739@gmail.com) | [shanhuhai5739](https://github.com/shanhuhai5739) | Kuaishou | | [Xiaofei Yu](mailto:nemo_00o@hotmail.com) | [xiaofei0800](https://github.com/xiaofei0800) | Baidu | -| [Wensi Yang](mailto:tianxinheihei@gmail.com) | [tianxinheihei](https://github.com/tianxinheihei) | Baidu | +| [Wensi Yang](mailto:tianxinheihei@gmail.com) | [tianxinheihei](https://github.com/tianxinheihei) | ByteDance | | [Kaiyu Zheng](mailto:412674752@qq.com) | [kaiyuzheng](https://github.com/kaiyuzheng) | ByteDance | +| [Yuqi Xiao](mailto:xiao19910705@163.com) | [Yuqi Xiao](https://github.com/YuqiXiao) | Baidu | diff --git a/VERSION b/VERSION index afaf360d3..1cc5f657e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 \ No newline at end of file +1.1.0 \ No newline at end of file diff --git a/bfe_balance/backend/bfe_backend.go b/bfe_balance/backend/bfe_backend.go index 969822a6f..75b2b79bb 100644 --- a/bfe_balance/backend/bfe_backend.go +++ b/bfe_balance/backend/bfe_backend.go @@ -41,6 +41,8 @@ type BfeBackend struct { succNum int // number of consecutive successes of health-check request closeChan chan bool // tell health-check to stop + + restarted bool // indicate if this backend is new bring-up by health-check } func NewBfeBackend() *BfeBackend { @@ -90,6 +92,19 @@ func (back *BfeBackend) setAvail(avail bool) { } } +func (back *BfeBackend) SetRestart(restart bool) { + back.Lock() + back.restarted = restart + back.Unlock() +} + +func (back *BfeBackend) GetRestart() bool { + back.RLock() + restart := back.restarted + back.RUnlock() + return restart +} + func (back *BfeBackend) ConnNum() int { back.RLock() connNum := back.connNum diff --git a/bfe_balance/backend/health_check.go b/bfe_balance/backend/health_check.go index 05f0c9ad1..c28b48542 100644 --- a/bfe_balance/backend/health_check.go +++ b/bfe_balance/backend/health_check.go @@ -96,6 +96,7 @@ loop: } log.Logger.Info("backend %s back to Normal", backend.Name) + backend.SetRestart(true) backend.SetAvail(true) break loop } diff --git a/bfe_balance/bal_gslb/bal_gslb.go b/bfe_balance/bal_gslb/bal_gslb.go index 4bd1e6c02..051d90442 100644 --- a/bfe_balance/bal_gslb/bal_gslb.go +++ b/bfe_balance/bal_gslb/bal_gslb.go @@ -89,6 +89,16 @@ func (bal *BalanceGslb) SetGslbBasic(gslbBasic cluster_conf.GslbBasicConf) { bal.lock.Unlock() } +func (bal *BalanceGslb) SetSlowStart(backendConf cluster_conf.BackendBasic) { + bal.lock.Lock() + + for _, sub := range bal.subClusters { + sub.setSlowStart(*backendConf.SlowStartTime) + } + + bal.lock.Unlock() +} + // Init inializes gslb cluster with config func (bal *BalanceGslb) Init(gslbConf gslb_conf.GslbClusterConf) error { totalWeight := 0 diff --git a/bfe_balance/bal_gslb/bal_gslb_test.go b/bfe_balance/bal_gslb/bal_gslb_test.go index 83fdf6b20..df09d6b37 100644 --- a/bfe_balance/bal_gslb/bal_gslb_test.go +++ b/bfe_balance/bal_gslb/bal_gslb_test.go @@ -133,3 +133,65 @@ func SetReqHeader(req *bfe_basic.Request, key string) { req.HttpRequest.Header.Set(key, "val") } } + +func TestSlowStart(t *testing.T) { + t.Logf("bal_gslb_test: TestSlowStart") + var c cluster_table_conf.ClusterBackend + var gb cluster_conf.GslbBasicConf + var g gslb_conf.GslbClusterConf + var err error + + loadJson("testdata/cluster1", &c) + loadJson("testdata/gb", &gb) + loadJson("testdata/g1", &g) + t.Logf("%v %v %v\n", c, gb, g) + + bal := NewBalanceGslb("cluster_dumi") + if err := bal.Init(g); err != nil { + t.Errorf("init error %s", err) + } + t.Logf("%+v\n", bal) + if bal.totalWeight != 100 || !bal.single || bal.subClusters[bal.avail].Name != "light.example.wt" || bal.retryMax != 3 || bal.crossRetry != 1 { + t.Errorf("init error") + } + + if len(bal.subClusters) != 3 { + t.Errorf("cluster len error") + } + + t.Logf("%+v", bal.subClusters[0]) + t.Logf("%+v", bal.subClusters[1]) + t.Logf("%+v", bal.subClusters[2]) + + var c1 cluster_table_conf.ClusterBackend + var gb1 cluster_conf.GslbBasicConf + var g1 gslb_conf.GslbClusterConf + loadJson("testdata/cluster2", &c1) + loadJson("testdata/gb2", &gb1) + loadJson("testdata/g2", &g1) + + err = cluster_conf.GslbBasicConfCheck(&gb1) + if err != nil { + t.Errorf("GslbBasicConfCheck err %s", err) + } + t.Logf("%v %v %v\n", c1, gb1, g1) + if err := bal.Reload(g1); err != nil { + t.Errorf("reload error %s", err) + } + + bal.SetGslbBasic(gb1) + + var backendConf cluster_conf.BackendBasic + err = cluster_conf.BackendBasicCheck(&backendConf) + if err != nil { + t.Errorf("BackendBasicCheck err %s", err) + } + var ssTime = 30 + backendConf.SlowStartTime = &ssTime + bal.SetSlowStart(backendConf) + + t.Logf("%+v\n", bal) + t.Logf("%+v", bal.subClusters[0]) + t.Logf("%+v", bal.subClusters[1]) + t.Logf("%+v", bal.subClusters[2]) +} diff --git a/bfe_balance/bal_gslb/sub_cluster.go b/bfe_balance/bal_gslb/sub_cluster.go index 6ea9c82c0..7498ba9ef 100644 --- a/bfe_balance/bal_gslb/sub_cluster.go +++ b/bfe_balance/bal_gslb/sub_cluster.go @@ -85,6 +85,10 @@ func (sub *SubCluster) balance(algor int, key []byte) (*backend.BfeBackend, erro return sub.backends.Balance(algor, key) } +func (sub *SubCluster) setSlowStart(slowStartTime int) { + sub.backends.SetSlowStart(slowStartTime) +} + // SubClusterList is a list of subcluster. type SubClusterList []*SubCluster diff --git a/bfe_balance/bal_slb/backend_rr.go b/bfe_balance/bal_slb/backend_rr.go index 54d79878b..1561aa042 100644 --- a/bfe_balance/bal_slb/backend_rr.go +++ b/bfe_balance/bal_slb/backend_rr.go @@ -16,15 +16,27 @@ package bal_slb +import ( + "time" +) + import ( "github.com/bfenetworks/bfe/bfe_balance/backend" "github.com/bfenetworks/bfe/bfe_config/bfe_cluster_conf/cluster_table_conf" ) +type WeightSS struct { + final int // final target weight after slow-start + slowStartTime int // time for backend increases the weight to the full value, in seconds + startTime time.Time // time of the first request +} + type BackendRR struct { - weight int // weight of this backend - current int // current weight - backend *backend.BfeBackend // point to BfeBackend + weight int // weight of this backend + current int // current weight + backend *backend.BfeBackend // point to BfeBackend + inSlowStart bool // indicate if in slow-start phase + weightSS WeightSS // slow_start related parameters } func NewBackendRR() *BackendRR { @@ -36,15 +48,17 @@ func NewBackendRR() *BackendRR { // Init initialize BackendRR with BackendConf func (backRR *BackendRR) Init(subClusterName string, conf *cluster_table_conf.BackendConf) { - backRR.weight = *conf.Weight - backRR.current = *conf.Weight + // scale up 100 times from conf file + backRR.weight = *conf.Weight * 100 + backRR.current = backRR.weight + backRR.weightSS.final = backRR.weight back := backRR.backend back.Init(subClusterName, conf) } func (backRR *BackendRR) UpdateWeight(weight int) { - backRR.weight = weight + backRR.weight = weight * 100 // if weight > 0, don't touch backRR.current if weight <= 0 { @@ -60,3 +74,33 @@ func (backRR *BackendRR) MatchAddrPort(addr string, port int) bool { back := backRR.backend return back.Addr == addr && back.Port == port } + +func (backRR *BackendRR) initSlowStart(ssTime int) { + backRR.weightSS.slowStartTime = ssTime + if backRR.weightSS.slowStartTime == 0 { + backRR.inSlowStart = false + } else { + backRR.weightSS.startTime = time.Now() + backRR.inSlowStart = true + + // set weight/current to 1, to avoid no traffic allowed at the beginning of start + backRR.weight = 1 + backRR.current = 1 + } +} + +func (backRR *BackendRR) updateSlowStart() { + if backRR.inSlowStart { + current := time.Duration(backRR.weightSS.final) * time.Since(backRR.weightSS.startTime) + if backRR.weightSS.slowStartTime != 0 { + current /= time.Duration(backRR.weightSS.slowStartTime) * time.Second + backRR.weight = int(current) + } else { + backRR.weight = backRR.weightSS.final + } + if backRR.weight >= backRR.weightSS.final { + backRR.weight = backRR.weightSS.final + backRR.inSlowStart = false + } + } +} diff --git a/bfe_balance/bal_slb/backend_rr_test.go b/bfe_balance/bal_slb/backend_rr_test.go index 96b1d19e6..82138d82c 100644 --- a/bfe_balance/bal_slb/backend_rr_test.go +++ b/bfe_balance/bal_slb/backend_rr_test.go @@ -36,12 +36,12 @@ func TestBackendRRInit_case1(t *testing.T) { backendRR := NewBackendRR() backendRR.Init("example.cluster", &conf) - if backendRR.weight != 10 { - t.Error("backend.weight should be 10") + if backendRR.weight != 10 * 100 { + t.Error("backend.weight should be 10 * 100") } - if backendRR.current != 10 { - t.Error("backend.current should be 10") + if backendRR.current != 10 * 100 { + t.Error("backend.current should be 10 * 100") } backend := backendRR.backend diff --git a/bfe_balance/bal_slb/bal_rr.go b/bfe_balance/bal_slb/bal_rr.go index fcd76dd31..b8b036a7e 100644 --- a/bfe_balance/bal_slb/bal_rr.go +++ b/bfe_balance/bal_slb/bal_rr.go @@ -89,10 +89,13 @@ func (s BackendListSorter) Less(i, j int) bool { type BalanceRR struct { sync.Mutex - Name string - backends BackendList // list of BackendRR - sorted bool // list of BackeneRR sorted or not - next int // next backend to schedule + Name string + backends BackendList // list of BackendRR + sorted bool // list of BackeneRR sorted or not + next int // next backend to schedule + + slowStartNum int // number of backends in slow_start phase + slowStartTime int // time for backend increases the weight to the full value, in seconds } func NewBalanceRR(name string) *BalanceRR { @@ -113,6 +116,27 @@ func (brr *BalanceRR) Init(conf cluster_table_conf.SubClusterBackend) { brr.next = 0 } +func (brr *BalanceRR) SetSlowStart(ssTime int) { + brr.Lock() + brr.slowStartTime = ssTime + brr.Unlock() +} + +func (brr *BalanceRR) checkSlowStart() { + brr.Lock() + defer brr.Unlock() + if brr.slowStartTime > 0 { + for _, backendRR := range brr.backends { + backend := backendRR.backend + if backend.GetRestart() { + backend.SetRestart(false) + backendRR.initSlowStart(brr.slowStartTime) + } + backendRR.updateSlowStart() + } + } +} + // Release releases backend list. func (brr *BalanceRR) Release() { for _, back := range brr.backends { @@ -162,6 +186,8 @@ func (brr *BalanceRR) Update(conf cluster_table_conf.SubClusterBackend) { for _, bkConf := range confMap { backendRR := NewBackendRR() backendRR.Init(brr.Name, bkConf) + backend := backendRR.backend + backend.SetRestart(true) // add to backendsNew backendsNew = append(backendsNew, backendRR) } @@ -195,6 +221,10 @@ func (brr *BalanceRR) ensureSortedUnlocked() { // Balance select one backend from sub cluster in round robin manner. func (brr *BalanceRR) Balance(algor int, key []byte) (*backend.BfeBackend, error) { + // Slow start is not supported when session sticky is enabled + if algor != WrrSticky { + brr.checkSlowStart() + } switch algor { case WrrSimple: return brr.simpleBalance() diff --git a/bfe_balance/bal_slb/bal_rr_test.go b/bfe_balance/bal_slb/bal_rr_test.go index 39905183c..2d478a577 100644 --- a/bfe_balance/bal_slb/bal_rr_test.go +++ b/bfe_balance/bal_slb/bal_rr_test.go @@ -45,18 +45,18 @@ func prepareBalanceRR() *BalanceRR { rr := &BalanceRR{ backends: []*BackendRR{ { - weight: 3, - current: 3, + weight: 300, + current: 300, backend: b1, }, { - weight: 2, - current: 2, + weight: 200, + current: 200, backend: b2, }, { - weight: 1, - current: 1, + weight: 100, + current: 100, backend: b3, }, }, @@ -80,11 +80,63 @@ func processBalance(t *testing.T, label string, algor int, key []byte, rr *Balan } } +func processSimpleBalance(t *testing.T, label string, algor int, key []byte, rr *BalanceRR, result []string) { + var l []string + loopCount := (300+200+100)+4 + + for i := 1; i < loopCount; i++ { + r, err := rr.Balance(algor, key) + if err != nil { + t.Errorf("should not error") + } + r.IncConnNum() + // append the end of backend b3 + if (i > 297) && (i <= 303) { + l = append(l, r.Name) + } + // append the end of backend b1 + if (i > 600) && (i <= 603) { + l = append(l, r.Name) + } + } + + if !reflect.DeepEqual(l, result) { + t.Errorf("balance error [%s] %v, expect %v", label, l, result) + } +} + +func processSimpleBalance3(t *testing.T, label string, algor int, key []byte, rr *BalanceRR, result []string) { + var l []string + loopCount := (200+100)*3+4 + + for i := 1; i < loopCount; i++ { + r, err := rr.Balance(algor, key) + if err != nil { + t.Errorf("should not error") + } + r.IncConnNum() + // append the end of backend b3 + if (i > 198) && (i <= 201) { + l = append(l, r.Name) + } + if (i > 498) && (i <= 501) { + l = append(l, r.Name) + } + if (i > 798) && (i <= 801) { + l = append(l, r.Name) + } + } + + if !reflect.DeepEqual(l, result) { + t.Errorf("balance error [%s] %v, expect %v", label, l, result) + } +} + func TestBalance(t *testing.T) { // case 1 rr := prepareBalanceRR() expectResult := []string{"b1", "b2", "b3", "b1", "b2", "b1", "b1", "b2", "b3"} - processBalance(t, "case 1", WrrSimple, nil, rr, expectResult) + processSimpleBalance(t, "case 1", WrrSimple, nil, rr, expectResult) // case 2 rr = prepareBalanceRR() @@ -95,7 +147,7 @@ func TestBalance(t *testing.T) { rr = prepareBalanceRR() rr.backends[0].backend.SetAvail(false) expectResult = []string{"b2", "b3", "b2", "b2", "b3", "b2", "b2", "b3", "b2"} - processBalance(t, "case 3", WrrSimple, nil, rr, expectResult) + processSimpleBalance3(t, "case 3", WrrSimple, nil, rr, expectResult) // case 4 rr = prepareBalanceRR() @@ -105,7 +157,7 @@ func TestBalance(t *testing.T) { // case 5 rr = prepareBalanceRR() - expectResult = []string{"b2", "b2", "b2", "b2", "b2", "b2", "b2", "b2", "b2"} + expectResult = []string{"b1", "b1", "b1", "b1", "b1", "b1", "b1", "b1", "b1"} processBalance(t, "case 5", WrrSticky, []byte{1}, rr, expectResult) rr.backends[0], rr.backends[2] = rr.backends[2], rr.backends[0] @@ -115,7 +167,9 @@ func TestBalance(t *testing.T) { // case 6 rr = prepareBalanceRR() rr.backends[0].backend.SetAvail(false) - expectResult = []string{"b2", "b2", "b2", "b2", "b2", "b2", "b2", "b2", "b2"} + // after scale up 100, the hash result changed + expectResult = []string{"b3", "b3", "b3", "b3", "b3", "b3", "b3", "b3", "b3"} +// expectResult = []string{"b2", "b2", "b2", "b2", "b2", "b2", "b2", "b2", "b2"} processBalance(t, "case 6", WrrSticky, []byte{1}, rr, expectResult) // case 7, lcw balance @@ -190,7 +244,7 @@ func checkBackend(t *testing.T, brr *BackendRR, name string, addr string, port i if b.Port != port { t.Errorf("backend port wrong, expect %d, actual %d", port, b.Port) } - if brr.weight != weight { + if brr.weight != weight*100 { t.Errorf("backend weight wrong, expect %d, actual %d", weight, brr.weight) } if connNum != -1 && b.ConnNum() != connNum { @@ -239,3 +293,9 @@ func BenchmarkStickyBalance(b *testing.B) { rr.stickyBalance(key) } } + +func TestSlowStart(t *testing.T) { + // case 1 + rr := prepareBalanceRR() + rr.SetSlowStart(30) +} diff --git a/bfe_balance/bal_table.go b/bfe_balance/bal_table.go index 046a49795..5f53ad9f2 100644 --- a/bfe_balance/bal_table.go +++ b/bfe_balance/bal_table.go @@ -188,6 +188,29 @@ func (t *BalTable) SetGslbBasic(clusterTable *bfe_route.ClusterTable) { } } +// SetSlowStart sets slow_start related conf (from server data conf) for BalTable. +// +// Note: +// - SetSlowStart() is called after server reload server data conf +// - SetSlowStart() should be concurrency safe +func (t *BalTable) SetSlowStart(clusterTable *bfe_route.ClusterTable) { + t.lock.RLock() + defer t.lock.RUnlock() + + if clusterTable == nil { + return + } + + for clusterName, bal := range t.balTable { + cluster, err := clusterTable.Lookup(clusterName) + if err != nil { + continue + } + + bal.SetSlowStart(*cluster.BackendConf()) + } +} + func (t *BalTable) BalTableReload(gslbConfs gslb_conf.GslbConf, backendConfs cluster_table_conf.ClusterTableConf) error { t.lock.Lock() diff --git a/bfe_config/bfe_cluster_conf/cluster_conf/cluster_conf_load.go b/bfe_config/bfe_cluster_conf/cluster_conf/cluster_conf_load.go index 47ad35747..a42894a8b 100644 --- a/bfe_config/bfe_cluster_conf/cluster_conf/cluster_conf_load.go +++ b/bfe_config/bfe_cluster_conf/cluster_conf/cluster_conf_load.go @@ -40,19 +40,6 @@ const ( DefaultReadClientAgainTimeout = 60000 ) -// Outlier detection levels -const ( - // Abnormal events about backend: - // - connect backend error - // - write request error(caused by backend) - // - read response header error - OutlierDetectionBasic = 0 - - // All abnormal events in basic level and: - // - response code is 5xx - OutlierDetection5XX = 1 -) - // HashStrategy for subcluster-level load balance (GSLB). // Note: // - CLIENTID is a special request header which represents a unique client, @@ -95,13 +82,14 @@ type FCGIConf struct { // BackendBasic is conf of backend basic type BackendBasic struct { - Protocol *string // backend protocol - TimeoutConnSrv *int // timeout for connect backend, in ms - TimeoutResponseHeader *int // timeout for read header from backend, in ms - MaxIdleConnsPerHost *int // max idle conns for each backend - RetryLevel *int // retry level if request fail - OutlierDetectionLevel *int // outlier detection level - + Protocol *string // backend protocol + TimeoutConnSrv *int // timeout for connect backend, in ms + TimeoutResponseHeader *int // timeout for read header from backend, in ms + MaxIdleConnsPerHost *int // max idle conns for each backend + MaxConnsPerHost *int // max conns for each backend (zero means unrestricted) + RetryLevel *int // retry level if request fail + SlowStartTime *int // time for backend increases the weight to the full value, in seconds + OutlierDetectionHttpCode *string // outlier detection http status code // protocol specific configurations FCGIConf *FCGIConf } @@ -186,14 +174,28 @@ func BackendBasicCheck(conf *BackendBasic) error { conf.MaxIdleConnsPerHost = &defaultIdle } + if conf.MaxConnsPerHost == nil || *conf.MaxConnsPerHost < 0 { + defaultConns := 0 + conf.MaxConnsPerHost = &defaultConns + } + if conf.RetryLevel == nil { retryLevel := RetryConnect conf.RetryLevel = &retryLevel } - if conf.OutlierDetectionLevel == nil { - outlierDetectionLevel := OutlierDetectionBasic - conf.OutlierDetectionLevel = &outlierDetectionLevel + if conf.OutlierDetectionHttpCode == nil { + outlierDetectionCode := "" + conf.OutlierDetectionHttpCode = &outlierDetectionCode + } else { + httpCode := *conf.OutlierDetectionHttpCode + httpCode = strings.ToLower(httpCode) + conf.OutlierDetectionHttpCode = &httpCode + } + + if conf.SlowStartTime == nil { + defaultSlowStartTime := 0 + conf.SlowStartTime = &defaultSlowStartTime } if conf.FCGIConf == nil { diff --git a/bfe_config/bfe_cluster_conf/cluster_conf/testdata/cluster_conf_1.conf b/bfe_config/bfe_cluster_conf/cluster_conf/testdata/cluster_conf_1.conf index 3f16d81c7..cc76434c8 100644 --- a/bfe_config/bfe_cluster_conf/cluster_conf/testdata/cluster_conf_1.conf +++ b/bfe_config/bfe_cluster_conf/cluster_conf/testdata/cluster_conf_1.conf @@ -6,7 +6,8 @@ "TimeoutConnSrv": 1000, "TimeoutWriteSrv": 2000, "TimeoutReadSrv": 2000, - "TimeoutResponseHeader":1000 + "TimeoutResponseHeader":1000, + "SlowStartTime": 30 }, "CheckConf": { "Uri": "/health", @@ -30,7 +31,8 @@ "TimeoutConnSrv": 1000, "TimeoutWriteSrv": 2000, "TimeoutReadSrv": 2000, - "TimeoutResponseHeader":1000 + "TimeoutResponseHeader":1000, + "SlowStartTime": 0 }, "CheckConf": { "Uri": "/health", diff --git a/bfe_config/bfe_conf/bfe_config_load.go b/bfe_config/bfe_conf/bfe_config_load.go index 7adc45e8b..2dfc13ef4 100644 --- a/bfe_config/bfe_conf/bfe_config_load.go +++ b/bfe_config/bfe_conf/bfe_config_load.go @@ -34,7 +34,6 @@ type BfeConfig struct { func SetDefaultConf(conf *BfeConfig) { conf.Server.SetDefaultConf() - conf.HttpsBasic.SetDefaultConf() conf.SessionCache.SetDefaultConf() conf.SessionTicket.SetDefaultConf() } diff --git a/bfe_config/bfe_conf/conf_https_basic.go b/bfe_config/bfe_conf/conf_https_basic.go index 8e63f24a2..32b4a1395 100644 --- a/bfe_config/bfe_conf/conf_https_basic.go +++ b/bfe_config/bfe_conf/conf_https_basic.go @@ -82,38 +82,12 @@ type ConfigHttpsBasic struct { ClientCRLBaseDir string // client cert CRL base directory } -func (cfg *ConfigHttpsBasic) SetDefaultConf() { - cfg.ServerCertConf = "tls_conf/server_cert_conf.data" - cfg.TlsRuleConf = "tls_conf/tls_rule_conf.data" - - cfg.CipherSuites = []string{ - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256|TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256|TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256|TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", - "TLS_ECDHE_RSA_WITH_RC4_128_SHA", - "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", - "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", - "TLS_RSA_WITH_RC4_128_SHA", - "TLS_RSA_WITH_AES_128_CBC_SHA", - "TLS_RSA_WITH_AES_256_CBC_SHA", - } - cfg.CurvePreferences = []string{ - "CurveP256", - } - - cfg.EnableSslv2ClientHello = true - - cfg.ClientCABaseDir = "tls_conf/client_ca" -} - func (cfg *ConfigHttpsBasic) Check(confRoot string) error { - // check cert file conf err := certConfCheck(cfg, confRoot) if err != nil { return err } - // check cert rule conf err = certRuleCheck(cfg, confRoot) if err != nil { return err @@ -129,24 +103,16 @@ func (cfg *ConfigHttpsBasic) Check(confRoot string) error { return err } - // check CipherSuites - for _, cipherGroup := range cfg.CipherSuites { - ciphers := strings.Split(cipherGroup, EquivCipherSep) - for _, cipher := range ciphers { - if _, ok := CipherSuitesMap[cipher]; !ok { - return fmt.Errorf("cipher (%s) not support", cipher) - } - } + err = cipherSuitesCheck(cfg) + if err != nil { + return err } - // check CurvePreferences - for _, curve := range cfg.CurvePreferences { - if _, ok := CurvesMap[curve]; !ok { - return fmt.Errorf("curve (%s) not support", curve) - } + err = curvePreferencesCheck(cfg) + if err != nil { + return err } - // check tls version err = tlsVersionCheck(cfg) if err != nil { return err @@ -169,6 +135,51 @@ func certConfCheck(cfg *ConfigHttpsBasic, confRoot string) error { return nil } +func cipherSuitesCheck(cfg *ConfigHttpsBasic) error { + if len(cfg.CipherSuites) == 0 { + log.Logger.Warn("CipherSuites not set, use default value") + cfg.CipherSuites = []string{ + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256|TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256|TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256|TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", + "TLS_ECDHE_RSA_WITH_RC4_128_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", + "TLS_RSA_WITH_RC4_128_SHA", + "TLS_RSA_WITH_AES_128_CBC_SHA", + "TLS_RSA_WITH_AES_256_CBC_SHA", + } + } + + for _, cipherGroup := range cfg.CipherSuites { + ciphers := strings.Split(cipherGroup, EquivCipherSep) + for _, cipher := range ciphers { + if _, ok := CipherSuitesMap[cipher]; !ok { + return fmt.Errorf("cipher (%s) not support", cipher) + } + } + } + + return nil +} + +func curvePreferencesCheck(cfg *ConfigHttpsBasic) error { + if len(cfg.CurvePreferences) == 0 { + log.Logger.Warn("CurvePreferences not set, use default value") + cfg.CurvePreferences = []string{ + "CurveP256", + } + } + + for _, curve := range cfg.CurvePreferences { + if _, ok := CurvesMap[curve]; !ok { + return fmt.Errorf("curve (%s) not support", curve) + } + } + + return nil +} + func certRuleCheck(cfg *ConfigHttpsBasic, confRoot string) error { if cfg.TlsRuleConf == "" { log.Logger.Warn("TlsRuleConf not set, use default value") diff --git a/bfe_http/cookie.go b/bfe_http/cookie.go index c768f6196..1a23d9f81 100644 --- a/bfe_http/cookie.go +++ b/bfe_http/cookie.go @@ -20,7 +20,6 @@ package bfe_http import ( "bytes" - "fmt" "net" "strconv" "strings" @@ -243,9 +242,12 @@ func SetCookie(w ResponseWriter, cookie *Cookie) { // header (if other fields are set). func (c *Cookie) String() string { var b bytes.Buffer - fmt.Fprintf(&b, "%s=%s", sanitizeCookieName(c.Name), sanitizeCookieValue(c.Value)) + b.WriteString(sanitizeCookieName(c.Name)) + b.WriteRune('=') + b.WriteString(sanitizeCookieValue(c.Value)) if len(c.Path) > 0 { - fmt.Fprintf(&b, "; Path=%s", sanitizeCookiePath(c.Path)) + b.WriteString("; Path=") + b.WriteString(sanitizeCookiePath(c.Path)) } if len(c.Domain) > 0 { if validCookieDomain(c.Domain) { @@ -257,25 +259,32 @@ func (c *Cookie) String() string { if d[0] == '.' { d = d[1:] } - fmt.Fprintf(&b, "; Domain=%s", d) + b.WriteString("; Domain=") + b.WriteString(d) } else { log.Logger.Warn("net/http: invalid Cookie.Domain %q; dropping domain attribute", c.Domain) } } if c.Expires.Unix() > 0 { - fmt.Fprintf(&b, "; Expires=%s", c.Expires.UTC().Format(TimeFormat)) + b.WriteString("; Expires=") + b2 := b.Bytes() + b.Reset() + b.Write(c.Expires.UTC().AppendFormat(b2, TimeFormat)) } if c.MaxAge > 0 { - fmt.Fprintf(&b, "; Max-Age=%d", c.MaxAge) + b.WriteString("; Max-Age=") + b2 := b.Bytes() + b.Reset() + b.Write(strconv.AppendInt(b2, int64(c.MaxAge), 10)) } else if c.MaxAge < 0 { - fmt.Fprintf(&b, "; Max-Age=0") + b.WriteString("; Max-Age=0") } if c.HttpOnly { - fmt.Fprintf(&b, "; HttpOnly") + b.WriteString("; HttpOnly") } if c.Secure { - fmt.Fprintf(&b, "; Secure") + b.WriteString("; Secure") } switch c.SameSite { case SameSiteDefaultMode: @@ -295,25 +304,28 @@ func (c *Cookie) String() string { // // if filter isn't empty, only cookies of that name are returned func readCookies(h Header, filter string) []*Cookie { - cookies := []*Cookie{} - lines, ok := h["Cookie"] - if !ok { - return cookies + lines := h["Cookie"] + if len(lines) == 0 { + return []*Cookie{} } + cookies := make([]*Cookie, 0, len(lines)+strings.Count(lines[0], ";")) + for _, line := range lines { - parts := strings.Split(strings.TrimSpace(line), ";") - if len(parts) == 1 && parts[0] == "" { - continue - } - // Per-line attributes - parsedPairs := 0 - for i := 0; i < len(parts); i++ { - parts[i] = strings.TrimSpace(parts[i]) - if len(parts[i]) == 0 { + line = strings.TrimSpace(line) + + var part string + for len(line) > 0 { + if splitIndex := strings.Index(line, ";"); splitIndex > 0 { + part, line = line[:splitIndex], line[splitIndex+1:] + } else { + part, line = line, "" + } + part = strings.TrimSpace(part) + if len(part) == 0 { continue } - name, val := parts[i], "" + name, val := part, "" if j := strings.Index(name, "="); j >= 0 { name, val = name[:j], name[j+1:] } @@ -328,7 +340,6 @@ func readCookies(h Header, filter string) []*Cookie { continue } cookies = append(cookies, &Cookie{Name: name, Value: val}) - parsedPairs++ } } return cookies diff --git a/bfe_http/cookie_test.go b/bfe_http/cookie_test.go index 98959a295..0fffcda97 100644 --- a/bfe_http/cookie_test.go +++ b/bfe_http/cookie_test.go @@ -386,3 +386,24 @@ func TestDisableSanitize(t *testing.T) { } } } + +func BenchmarkCookieString(b *testing.B) { + const wantCookieString = `cookie-9=i3e01nf61b6t23bvfmplnanol3; Path=/restricted/; Domain=example.com; Expires=Tue, 10 Nov 2009 23:00:00 GMT; Max-Age=3600` + c := &Cookie{ + Name: "cookie-9", + Value: "i3e01nf61b6t23bvfmplnanol3", + Expires: time.Unix(1257894000, 0), + Path: "/restricted/", + Domain: ".example.com", + MaxAge: 3600, + } + var benchmarkCookieString string + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchmarkCookieString = c.String() + } + if have, want := benchmarkCookieString, wantCookieString; have != want { + b.Fatalf("Have: %v Want: %v", have, want) + } +} diff --git a/bfe_http/header.go b/bfe_http/header.go index f8642338d..f316a983b 100644 --- a/bfe_http/header.go +++ b/bfe_http/header.go @@ -59,6 +59,15 @@ func (h Header) Get(key string) string { return textproto.MIMEHeader(h).Get(key) } +// Values returns all values associated with the given key. +// It is case insensitive; textproto.CanonicalMIMEHeaderKey is +// used to canonicalize the provided key. To use non-canonical +// keys, access the map directly. +// The returned slice is not a copy. +func (h Header) Values(key string) []string { + return textproto.MIMEHeader(h).Values(key) +} + // GetDirect gets the value associated with the given key // in CanonicalHeaderKey form. func (h Header) GetDirect(key string) string { diff --git a/bfe_http/transport.go b/bfe_http/transport.go index 1c2bae3c1..3f129d176 100644 --- a/bfe_http/transport.go +++ b/bfe_http/transport.go @@ -69,6 +69,12 @@ type Transport struct { altMu sync.RWMutex altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper + connMu sync.Mutex // mutex for conn count + // Conn count which record current connection of each backend + // when create a persistConn we count plus one of the cm key, + // and minus one when the persistConn is close. + connCnt map[string]int + // Proxy specifies a function to return a proxy for a given // Request. If the function returns a non-nil error, the // request is aborted with the provided error. @@ -103,6 +109,10 @@ type Transport struct { // DefaultMaxIdleConnsPerHost is used. MaxIdleConnsPerHost int + // MaxConnsPerHost, if non-zero, controls the maximum currency conns + // to per-host. If less than or equal zero, transport will ignore this value. + MaxConnsPerHost int + // ResponseHeaderTimeout, if non-zero, specifies the amount of // time to wait for a server's response headers after fully // writing the request (including its body, if any). This @@ -321,6 +331,16 @@ func (cm *connectMethod) proxyAuth() string { return "" } +func (t *Transport) releaseConnCnt(cacheKey string) { + t.connMu.Lock() + if t.connCnt == nil { + t.connMu.Unlock() + return + } + t.connCnt[cacheKey]-- + t.connMu.Unlock() +} + // putIdleConn adds pconn to the list of idle persistent connections awaiting // a new request. // If pconn is no longer needed or not in a good state, putIdleConn @@ -443,6 +463,21 @@ func (t *Transport) dial(network, addr string) (c net.Conn, err error) { return net.Dial(network, addr) } +// check whether we can create new conn to backend with given cachekey +func (t *Transport) checkAndIncConnCnt(cacheKey string, maxValue int) bool { + t.connMu.Lock() + if t.connCnt == nil { + t.connCnt = make(map[string]int) + } + if val, ok := t.connCnt[cacheKey]; ok && val >= maxValue { + t.connMu.Unlock() + return false + } + t.connCnt[cacheKey]++ + t.connMu.Unlock() + return true +} + // getConn dials and creates a new persistConn to the target as // specified in the connectMethod. This includes doing a proxy CONNECT // and/or setting up TLS. If this doesn't return an error, the persistConn @@ -458,11 +493,19 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, error) { } dialc := make(chan dialRes) go func() { + cacheKey := cm.key() + if t.MaxConnsPerHost > 0 && !t.checkAndIncConnCnt(cacheKey, t.MaxConnsPerHost) { + dialc <- dialRes{nil, fmt.Errorf("cm key[%v] greater than max conns[%d]", cacheKey, t.MaxConnsPerHost)} + return + } pc, err := t.dialConn(cm) state.HttpBackendConnAll.Inc(1) if err == nil { state.HttpBackendConnSucc.Inc(1) + } else { + t.releaseConnCnt(cacheKey) } + dialc <- dialRes{pc, err} }() @@ -1035,6 +1078,9 @@ func (pc *persistConn) closeLocked() { if !pc.closed { pc.conn.Close() pc.closed = true + // there are some many reason to close a conn, in order to avoid missing release in some place, + // it is a safely way to release conn cnt in pc.close() + pc.t.releaseConnCnt(pc.cacheKey) } pc.mutateHeaderFunc = nil } diff --git a/bfe_http2/server.go b/bfe_http2/server.go index 60e387f82..9ccc0945c 100644 --- a/bfe_http2/server.go +++ b/bfe_http2/server.go @@ -1567,12 +1567,23 @@ func (sc *serverConn) processSettingInitialWindowSize(val uint32) error { func (sc *serverConn) processData(f *DataFrame) error { sc.serveG.Check() + id := f.Header().StreamID + if sc.inGoAway && (sc.goAwayCode != ErrCodeNo || id > sc.maxStreamID) { + // Discard all DATA frames if the GOAWAY is due to an + // error, or: + // + // Section 6.8: After sending a GOAWAY frame, the sender + // can discard frames for streams initiated by the + // receiver with identifiers higher than the identified + // last stream. + return nil + } + data := f.Data() // "If a DATA frame is received whose stream is not in "open" // or "half closed (local)" state, the recipient MUST respond // with a stream error (Section 5.4.2) of type STREAM_CLOSED." - id := f.Header().StreamID st, ok := sc.streams[id] if !ok || st.state != stateOpen || st.gotTrailerHeader { // This includes sending a RST_STREAM if the stream is @@ -1607,7 +1618,10 @@ func (sc *serverConn) processData(f *DataFrame) error { if st.declBodyBytes != -1 && st.bodyBytes+int64(len(data)) > st.declBodyBytes { err := fmt.Errorf("sender tried to send more than declared Content-Length of %d bytes", st.declBodyBytes) st.body.CloseWithError(err) - return StreamError{id, ErrCodeStreamClosed, err.Error()} + // RFC 7540, sec 8.1.2.6: A request or response is also malformed if the + // value of a content-length header field does not equal the sum of the + // DATA frame payload lengths that form the body. + return StreamError{id, ErrCodeProtocol, err.Error()} } if f.Length > 0 { // Check whether the client has flow control quota. diff --git a/bfe_http2/server_test.go b/bfe_http2/server_test.go index 57fe049be..a37650f85 100644 --- a/bfe_http2/server_test.go +++ b/bfe_http2/server_test.go @@ -2960,3 +2960,68 @@ y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== -----END RSA PRIVATE KEY-----`) + +func TestNoRstPostAfterGOAWAY(t *testing.T) { + const msg = "Hello, world." + st := newServerTester(t, func(w http.ResponseWriter, r *http.Request) { + n, err := io.Copy(ioutil.Discard, r.Body) + if err != nil || n > 0 { + t.Errorf("Read %d bytes, error %v; want 0 bytes.", n, err) + } + io.WriteString(w, msg) + }) + defer st.Close() + st.greet() + // Give the server quota to reply. (plus it has the the 64KB) + if err := st.fr.WriteWindowUpdate(0, uint32(1*len(msg))); err != nil { + t.Fatal(err) + } + hbf := st.encodeHeader(":method", "POST") + st.writeHeaders(HeadersFrameParam{ + StreamID: 1, + BlockFragment: hbf, + EndStream: false, + EndHeaders: true, + }) + close(st.sc.closeNotifyCh) + st.writeData(1, true, nil) + + st.wantGoAway() + for { + f, err := st.readFrame() + if err == io.EOF { + st.t.Fatal("got a EOF; want *GoAwayFrame") + } + if err != nil && err.Error() == "timeout waiting for frame" { + break + } + if err != nil { + t.Fatal(err) + } + if gf, ok := f.(*RSTStreamFrame); ok && gf.StreamID == 1 { + t.Fatal("got rst but want no ret") + break + } + } + +} + +func TestServer_Rejects_TooSmall(t *testing.T) { + testServerResponse(t, func(w http.ResponseWriter, r *http.Request) error { + ioutil.ReadAll(r.Body) + return nil + }, func(st *serverTester) { + st.writeHeaders(HeadersFrameParam{ + StreamID: 1, // clients send odd numbers + BlockFragment: st.encodeHeader( + ":method", "POST", + "content-length", "4", + ), + EndStream: false, // to say DATA frames are coming + EndHeaders: true, + }) + st.writeData(1, true, []byte("12345")) + + st.wantRSTStream(1, ErrCodeProtocol) + }) +} diff --git a/bfe_modules/mod_header/action.go b/bfe_modules/mod_header/action.go index f0d0a55f4..b302a9158 100644 --- a/bfe_modules/mod_header/action.go +++ b/bfe_modules/mod_header/action.go @@ -51,8 +51,10 @@ func ActionFileCheck(conf ActionFile) error { switch *conf.Cmd { case "REQ_HEADER_SET", "REQ_HEADER_ADD", + "REQ_HEADER_RENAME", "RSP_HEADER_SET", - "RSP_HEADER_ADD": + "RSP_HEADER_ADD", + "RSP_HEADER_RENAME": // header and value if len(conf.Params) != 2 { @@ -193,7 +195,7 @@ func expectPercent(str string) int { return index } -const variableCharset = "abcdefghijklmnopqrstuvwxyz_" +const variableCharset = "abcdefghijklmnopqrstuvwxyz0123456789_" func expectVariableParam(str string) int { index := 0 @@ -374,7 +376,11 @@ func actionConvert(actionFile ActionFile) (Action, error) { // append key values action.Params = append(action.Params, key) action.Params = append(action.Params, values...) - + case "REQ_HEADER_RENAME", "RSP_HEADER_RENAME": + originalKey := textproto.CanonicalMIMEHeaderKey(actionFile.Params[0]) + newKey := textproto.CanonicalMIMEHeaderKey(actionFile.Params[1]) + action.Params = append(action.Params, originalKey) + action.Params = append(action.Params, newKey) case "REQ_HEADER_DEL", "RSP_HEADER_DEL": // - REQ_HEADER_DEL: [referer] // - RSP_HEADER_DEL: [location] @@ -430,6 +436,8 @@ func HeaderActionDo(h *bfe_http.Header, cmd string, headerName string, value str // delete case "HEADER_DEL": headerDel(h, headerName) + case "HEADER_RENAME": + headerRename(h, headerName, value) } } @@ -447,26 +455,35 @@ func getHeader(req *bfe_basic.Request, headerType int) (h *bfe_http.Header) { func processHeader(req *bfe_basic.Request, headerType int, action Action) { var key string var value string + var cmd string h := getHeader(req, headerType) - if action.Cmd[4:] == "HEADER_MOD" { + cmd = action.Cmd[4:] + + switch cmd { + case "HEADER_MOD": key = action.Params[1] // get header value if value = h.Get(key); value == "" { // if req do not have this header, continue return } - // mod header value value = modHeaderValue(value, action) - } else { + case "HEADER_RENAME": + originalKey, newKey := action.Params[0], action.Params[1] + if h.Get(originalKey) == "" || h.Get(newKey) != "" { + return + } + key, value = originalKey, newKey + default: key = action.Params[0] value = getHeaderValue(req, action) } // trim action.Cmd prefix REQ_ and RSP_ - HeaderActionDo(h, action.Cmd[4:], key, value) + HeaderActionDo(h, cmd, key, value) } func processCookie(req *bfe_basic.Request, headerType int, action Action) { diff --git a/bfe_modules/mod_header/action_header.go b/bfe_modules/mod_header/action_header.go index cabe273f7..906b35102 100644 --- a/bfe_modules/mod_header/action_header.go +++ b/bfe_modules/mod_header/action_header.go @@ -32,3 +32,10 @@ func headerAdd(h *bfe_http.Header, key string, value string) { func headerDel(h *bfe_http.Header, key string) { h.Del(key) } + +// rename header originalKey to newKey +func headerRename(h *bfe_http.Header, originalKey, newKey string) { + val := h.Get(originalKey) + h.Set(newKey, val) + h.Del(originalKey) +} diff --git a/bfe_modules/mod_header/action_header_var.go b/bfe_modules/mod_header/action_header_var.go index cc864c488..ac5460063 100644 --- a/bfe_modules/mod_header/action_header_var.go +++ b/bfe_modules/mod_header/action_header_var.go @@ -62,6 +62,8 @@ var VariableHandlers = map[string]HeaderValueHandler{ "bfe_ssl_resume": getBfeSslResume, "bfe_ssl_cipher": getBfeSslCipher, "bfe_ssl_version": getBfeSslVersion, + "bfe_ssl_ja3_raw": getBfeSslJa3Raw, + "bfe_ssl_ja3_hash": getBfeSslJa3Hash, "bfe_protocol": getBfeProtocol, "client_cert_serial_number": getClientCertSerialNumber, "client_cert_subject_title": getClientCertSubjectTitle, @@ -180,6 +182,24 @@ func getBfeSslVersion(req *bfe_basic.Request) string { return bfe_tls.VersionTextForOpenSSL(state.Version) } +// get tls ja3 string +func getBfeSslJa3Raw(req *bfe_basic.Request) string { + if req.Session.TlsState == nil { + return "" + } + state := req.Session.TlsState + return state.JA3Raw +} + +// get tls ja3 hash +func getBfeSslJa3Hash(req *bfe_basic.Request) string { + if req.Session.TlsState == nil { + return "" + } + state := req.Session.TlsState + return state.JA3Hash +} + // get protocol for application level func getBfeProtocol(req *bfe_basic.Request) string { return req.Protocol() diff --git a/bfe_modules/mod_header/action_test.go b/bfe_modules/mod_header/action_test.go index 8018e8fc4..8c1e1454c 100644 --- a/bfe_modules/mod_header/action_test.go +++ b/bfe_modules/mod_header/action_test.go @@ -165,6 +165,27 @@ func TestHeaderActionsDo_Case4(t *testing.T) { } } +func TestHeaderActionsDo_Case5(t *testing.T) { + req := makeBasicRequest("http://www.example.org") + + cmdMod := "REQ_HEADER_RENAME" + action := Action{Cmd: cmdMod, Params: []string{"OriginalKey", "NewKey"}} + expectVal := "TestCase" + + req.HttpRequest.Header.Add("OriginalKey", expectVal) + HeaderActionsDo(req, 0, []Action{action}) + + value := req.HttpRequest.Header.Get("NewKey") + if value != expectVal { + t.Errorf("header rename newkey want[%s] got[%s]", expectVal, value) + } + + value = req.HttpRequest.Header.Get("OriginalKey") + if value != "" { + t.Errorf("header rename originalkey want[%s] got[%s]", "", value) + } +} + func TestActionsConvert(t *testing.T) { cmdSet := "REQ_HEADER_SET" cmdAdd := "REQ_HEADER_ADD" diff --git a/bfe_modules/mod_header/testdata/mod_header/header_rule.data b/bfe_modules/mod_header/testdata/mod_header/header_rule.data index 46c494953..6fe18d256 100644 --- a/bfe_modules/mod_header/testdata/mod_header/header_rule.data +++ b/bfe_modules/mod_header/testdata/mod_header/header_rule.data @@ -2,6 +2,19 @@ "Version": "1234", "Config": { "p1": [ + { + "cond": "req_path_prefix_in(\"/header_rename\", false)", + "actions": [ + { + "cmd": "REQ_HEADER_RENAME", + "params": [ + "OriginalKey", + "NewKey" + ] + } + ], + "last": true + }, { "cond": "req_path_prefix_in(\"/cookie_set\", false)", "actions": [ @@ -183,7 +196,7 @@ ] } ], - "last": true + "last": false } ] } diff --git a/bfe_net/textproto/header.go b/bfe_net/textproto/header.go index 84c2b4b32..d6db58303 100644 --- a/bfe_net/textproto/header.go +++ b/bfe_net/textproto/header.go @@ -54,6 +54,18 @@ func (h MIMEHeader) Get(key string) string { return v[0] } +// Values returns all values associated with the given key. +// It is case insensitive; CanonicalMIMEHeaderKey is +// used to canonicalize the provided key. To use non-canonical +// keys, access the map directly. +// The returned slice is not a copy. +func (h MIMEHeader) Values(key string) []string { + if h == nil { + return nil + } + return h[CanonicalMIMEHeaderKey(key)] +} + // Del deletes the values associated with key. func (h MIMEHeader) Del(key string) { delete(h, CanonicalMIMEHeaderKey(key)) diff --git a/bfe_net/textproto/header_test.go b/bfe_net/textproto/header_test.go new file mode 100644 index 000000000..e800aede9 --- /dev/null +++ b/bfe_net/textproto/header_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2021 The BFE Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package textproto + +import "testing" + +type canonicalHeaderKeyTest struct { + in, out string +} + +var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{ + {"a-b-c", "A-B-C"}, + {"a-1-c", "A-1-C"}, + {"User-Agent", "User-Agent"}, + {"uSER-aGENT", "User-Agent"}, + {"user-agent", "User-Agent"}, + {"USER-AGENT", "User-Agent"}, + + // Other valid tchar bytes in tokens: + {"foo-bar_baz", "Foo-Bar_baz"}, + {"foo-bar$baz", "Foo-Bar$baz"}, + {"foo-bar~baz", "Foo-Bar~baz"}, + {"foo-bar*baz", "Foo-Bar*baz"}, + + // Non-ASCII or anything with spaces or non-token chars is unchanged: + {"üser-agenT", "üser-agenT"}, + {"a B", "a B"}, + + // This caused a panic due to mishandling of a space: + {"C Ontent-Transfer-Encoding", "C Ontent-Transfer-Encoding"}, + {"foo bar", "foo bar"}, +} + +func TestCanonicalMIMEHeaderKey(t *testing.T) { + for _, tt := range canonicalHeaderKeyTests { + if s := CanonicalMIMEHeaderKey(tt.in); s != tt.out { + t.Errorf("CanonicalMIMEHeaderKey(%q) = %q, want %q", tt.in, s, tt.out) + } + } +} + +// Issue #34799 add a Header method to get multiple values []string, with canonicalized key +func TestMIMEHeaderMultipleValues(t *testing.T) { + testHeader := MIMEHeader{ + "Set-Cookie": {"cookie 1", "cookie 2"}, + } + values := testHeader.Values("set-cookie") + n := len(values) + if n != 2 { + t.Errorf("count: %d; want 2", n) + } +} diff --git a/bfe_net/textproto/pipeline.go b/bfe_net/textproto/pipeline.go index a10d294fd..5db2001d3 100644 --- a/bfe_net/textproto/pipeline.go +++ b/bfe_net/textproto/pipeline.go @@ -86,7 +86,7 @@ func (p *Pipeline) EndResponse(id uint) { type sequencer struct { mu sync.Mutex id uint - wait map[uint]chan uint + wait map[uint]chan struct{} } // Start waits until it is time for the event numbered id to begin. @@ -98,9 +98,9 @@ func (s *sequencer) Start(id uint) { s.mu.Unlock() return } - c := make(chan uint) + c := make(chan struct{}) if s.wait == nil { - s.wait = make(map[uint]chan uint) + s.wait = make(map[uint]chan struct{}) } s.wait[id] = c s.mu.Unlock() @@ -113,12 +113,13 @@ func (s *sequencer) Start(id uint) { func (s *sequencer) End(id uint) { s.mu.Lock() if s.id != id { + s.mu.Unlock() panic("out of sync") } id++ s.id = id if s.wait == nil { - s.wait = make(map[uint]chan uint) + s.wait = make(map[uint]chan struct{}) } c, ok := s.wait[id] if ok { @@ -126,6 +127,6 @@ func (s *sequencer) End(id uint) { } s.mu.Unlock() if ok { - c <- 1 + close(c) } } diff --git a/bfe_net/textproto/reader.go b/bfe_net/textproto/reader.go index e4eedb5d5..153e6c664 100644 --- a/bfe_net/textproto/reader.go +++ b/bfe_net/textproto/reader.go @@ -148,12 +148,13 @@ func (r *Reader) readContinuedLineSlice() ([]byte, error) { } // Optimistically assume that we have started to buffer the next line - // and it starts with an ASCII letter (the next header key), so we can - // avoid copying that buffered data around in memory and skipping over - // non-existent whitespace. + // and it starts with an ASCII letter (the next header key), or a blank + // line, so we can avoid copying that buffered data around in memory + // and skipping over non-existent whitespace. if r.R.Buffered() > 1 { - peek, err := r.R.Peek(1) - if err == nil && isASCIILetter(peek[0]) { + peek, _ := r.R.Peek(2) + if len(peek) > 0 && (isASCIILetter(peek[0]) || peek[0] == '\n') || + len(peek) == 2 && peek[0] == '\r' && peek[1] == '\n' { return trim(line), nil } } @@ -169,7 +170,7 @@ func (r *Reader) readContinuedLineSlice() ([]byte, error) { break } r.buf = append(r.buf, ' ') - r.buf = append(r.buf, line...) + r.buf = append(r.buf, trim(line)...) } return r.buf, nil } diff --git a/bfe_net/textproto/reader_test.go b/bfe_net/textproto/reader_test.go index 42f1dce8a..0b1632704 100644 --- a/bfe_net/textproto/reader_test.go +++ b/bfe_net/textproto/reader_test.go @@ -30,41 +30,6 @@ import ( "github.com/bfenetworks/bfe/bfe_bufio" ) -type canonicalHeaderKeyTest struct { - in, out string -} - -var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{ - {"a-b-c", "A-B-C"}, - {"a-1-c", "A-1-C"}, - {"User-Agent", "User-Agent"}, - {"uSER-aGENT", "User-Agent"}, - {"user-agent", "User-Agent"}, - {"USER-AGENT", "User-Agent"}, - - // Other valid tchar bytes in tokens: - {"foo-bar_baz", "Foo-Bar_baz"}, - {"foo-bar$baz", "Foo-Bar$baz"}, - {"foo-bar~baz", "Foo-Bar~baz"}, - {"foo-bar*baz", "Foo-Bar*baz"}, - - // Non-ASCII or anything with spaces or non-token chars is unchanged: - {"üser-agenT", "üser-agenT"}, - {"a B", "a B"}, - - // This caused a panic due to mishandling of a space: - {"C Ontent-Transfer-Encoding", "C Ontent-Transfer-Encoding"}, - {"foo bar", "foo bar"}, -} - -func TestCanonicalMIMEHeaderKey(t *testing.T) { - for _, tt := range canonicalHeaderKeyTests { - if s := CanonicalMIMEHeaderKey(tt.in); s != tt.out { - t.Errorf("CanonicalMIMEHeaderKey(%q) = %q, want %q", tt.in, s, tt.out) - } - } -} - func reader(s string) *Reader { return NewReader(bfe_bufio.NewReader(strings.NewReader(s))) } @@ -226,6 +191,32 @@ func TestReadMIMEHeaderNonCompliant(t *testing.T) { } } +// Test that continued lines are properly trimmed. Issue 11204. +func TestReadMIMEHeaderTrimContinued(t *testing.T) { + // In this header, \n and \r\n terminated lines are mixed on purpose. + // We expect each line to be trimmed (prefix and suffix) before being concatenated. + // Keep the spaces as they are. + r := reader("" + // for code formatting purpose. + "a:\n" + + " 0 \r\n" + + "b:1 \t\r\n" + + "c: 2\r\n" + + " 3\t\n" + + " \t 4 \r\n\n") + m, err := r.ReadMIMEHeader() + if err != nil { + t.Fatal(err) + } + want := MIMEHeader{ + "A": {"0"}, + "B": {"1"}, + "C": {"2 3 4"}, + } + if !reflect.DeepEqual(m, want) { + t.Fatalf("ReadMIMEHeader mismatch.\n got: %q\nwant: %q", m, want) + } +} + type readResponseTest struct { in string inCode int diff --git a/bfe_net/textproto/writer.go b/bfe_net/textproto/writer.go index 118dbb878..2d23e337b 100644 --- a/bfe_net/textproto/writer.go +++ b/bfe_net/textproto/writer.go @@ -75,7 +75,8 @@ type dotWriter struct { } const ( - wstateBeginLine = iota // beginning of line; initial state; must be zero + wstateBegin = iota // initial state; must be zero + wstateBeginLine // beginning of line wstateCR // wrote \r (possibly at end of line) wstateData // writing data in middle of line ) @@ -85,7 +86,7 @@ func (d *dotWriter) Write(b []byte) (n int, err error) { for n < len(b) { c := b[n] switch d.state { - case wstateBeginLine: + case wstateBegin, wstateBeginLine: d.state = wstateData if c == '.' { // escape leading dot diff --git a/bfe_net/textproto/writer_test.go b/bfe_net/textproto/writer_test.go index b7056257b..70cba2a2c 100644 --- a/bfe_net/textproto/writer_test.go +++ b/bfe_net/textproto/writer_test.go @@ -50,3 +50,29 @@ func TestDotWriter(t *testing.T) { t.Fatalf("wrote %q", s) } } + +func TestDotWriterCloseEmptyWrite(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(bfe_bufio.NewWriter(&buf)) + d := w.DotWriter() + n, err := d.Write([]byte{}) + if n != 0 || err != nil { + t.Fatalf("Write: %d, %s", n, err) + } + d.Close() + want := "\r\n.\r\n" + if s := buf.String(); s != want { + t.Fatalf("wrote %q; want %q", s, want) + } +} + +func TestDotWriterCloseNoWrite(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(bfe_bufio.NewWriter(&buf)) + d := w.DotWriter() + d.Close() + want := "\r\n.\r\n" + if s := buf.String(); s != want { + t.Fatalf("wrote %q; want %q", s, want) + } +} diff --git a/bfe_route/bfe_cluster/bfe_cluster.go b/bfe_route/bfe_cluster/bfe_cluster.go index 095ea7071..19eb2bb31 100644 --- a/bfe_route/bfe_cluster/bfe_cluster.go +++ b/bfe_route/bfe_cluster/bfe_cluster.go @@ -113,12 +113,11 @@ func (cluster *BfeCluster) RetryLevel() int { return *retryLevel } -func (cluster *BfeCluster) OutlierDetectionLevel() int { +func (cluster *BfeCluster) OutlierDetectionHttpCode() string { cluster.RLock() - outlierDetectionLevel := cluster.backendConf.OutlierDetectionLevel + outlierDetectionHttpCode := cluster.backendConf.OutlierDetectionHttpCode cluster.RUnlock() - - return *outlierDetectionLevel + return *outlierDetectionHttpCode } func (cluster *BfeCluster) TimeoutReadClient() time.Duration { diff --git a/bfe_server/bfe_confdata_load.go b/bfe_server/bfe_confdata_load.go index c44cf1e0c..ae99a2112 100644 --- a/bfe_server/bfe_confdata_load.go +++ b/bfe_server/bfe_confdata_load.go @@ -56,10 +56,11 @@ func (srv *BfeServer) InitDataLoad() error { return fmt.Errorf("InitDataLoad():balTableInit Error %s", err) } - // set gslb retry config + // set gslb retry config, slow_start config if srv.ServerConf != nil { ct := srv.ServerConf.ClusterTable srv.balTable.SetGslbBasic(ct) + srv.balTable.SetSlowStart(ct) } log.Logger.Info("init bal table success") @@ -116,6 +117,8 @@ func (srv *BfeServer) serverDataConfReload(hostFile, vipFile, routeFile, cluster // set gslb basic srv.balTable.SetGslbBasic(newServerConf.ClusterTable) + // set slow_start config + srv.balTable.SetSlowStart(newServerConf.ClusterTable) return nil } @@ -152,6 +155,8 @@ func (srv *BfeServer) gslbDataConfReload(gslbFile, clusterTableFile string) erro serverConf := srv.ServerConf srv.confLock.Unlock() srv.balTable.SetGslbBasic(serverConf.ClusterTable) + // set slow_start config + srv.balTable.SetSlowStart(serverConf.ClusterTable) return nil } diff --git a/bfe_server/reverseproxy.go b/bfe_server/reverseproxy.go index 29af312b2..fdfc954e1 100644 --- a/bfe_server/reverseproxy.go +++ b/bfe_server/reverseproxy.go @@ -25,6 +25,8 @@ import ( "io" "net" "reflect" + "strconv" + "strings" "sync" "time" ) @@ -155,6 +157,7 @@ func (p *ReverseProxy) setTransports(clusterMap bfe_route.ClusterMap) { // get transport, check if transport needs update backendConf := conf.BackendConf() if (t.MaxIdleConnsPerHost != *backendConf.MaxIdleConnsPerHost) || + (t.MaxConnsPerHost != *backendConf.MaxConnsPerHost) || (t.ResponseHeaderTimeout != time.Millisecond*time.Duration(*backendConf.TimeoutResponseHeader)) || (t.ReqWriteBufferSize != conf.ReqWriteBufferSize()) || (t.ReqFlushInterval != conf.ReqFlushInterval()) { @@ -215,6 +218,7 @@ func createTransport(cluster *bfe_cluster.BfeCluster) bfe_http.RoundTripper { ReqWriteBufferSize: cluster.ReqWriteBufferSize(), ReqFlushInterval: cluster.ReqFlushInterval(), DisableCompression: true, + MaxConnsPerHost: *backendConf.MaxConnsPerHost, } case "fcgi": return &bfe_fcgi.Transport{ @@ -334,7 +338,7 @@ func (p *ReverseProxy) clusterInvoke(srv *BfeServer, cluster *bfe_cluster.BfeClu request.Backend.BackendPort = uint32(backend.Port) if err == nil { - if checkBackendStatus(cluster.OutlierDetectionLevel(), res.StatusCode) { + if checkBackendStatus(cluster.OutlierDetectionHttpCode(), res.StatusCode) { backend.OnFail(cluster.Name) } else { backend.OnSuccess() @@ -758,8 +762,16 @@ response_got: // we must timeout both conns after specified duration. p.setTimeout(bfe_basic.StageWriteClient, basicReq.Connection, req, timeoutWriteClient) writeTimer = time.AfterFunc(timeoutWriteClient, func() { - transport := basicReq.Trans.Transport.(*bfe_http.Transport) - transport.CancelRequest(basicReq.OutRequest) // force close connection to backend + if basicReq.Trans.Transport != nil { + // TODO: process bfe_fcgi.Transport & bfe_http2.Transport + switch t := basicReq.Trans.Transport.(type) { + case *bfe_http.Transport: + t.CancelRequest(req) + default: + // do nothing + } + } + }) defer writeTimer.Stop() @@ -878,6 +890,25 @@ func checkRequestWithoutBody(req *bfe_http.Request) bool { return false } -func checkBackendStatus(outlierDetectionLevel int, statusCode int) bool { - return outlierDetectionLevel == cluster_conf.OutlierDetection5XX && statusCode/100 == 5 +func checkBackendStatus(outlierDetectionHttpCodeStr string, statusCode int) bool { + if outlierDetectionHttpCodeStr == "" { + return false + } + for _, code := range strings.Split(outlierDetectionHttpCodeStr, "|") { + switch code { + case "3xx", "4xx", "5xx": + if strconv.Itoa(statusCode/100) == code[0:1] { + return true + } + default: + codeInt, err := strconv.Atoi(code) + if err != nil { + continue + } + if codeInt == statusCode { + return true + } + } + } + return false } diff --git a/bfe_spdy/frame_test.go b/bfe_spdy/frame_test.go index 28020a5e0..cbf73d89e 100644 --- a/bfe_spdy/frame_test.go +++ b/bfe_spdy/frame_test.go @@ -538,10 +538,12 @@ func TestMultipleSPDYFrames(t *testing.T) { // Start the goroutines to write the frames. go func() { if err := writer.WriteFrame(&headersFrame); err != nil { - t.Fatal("WriteFrame (HEADERS): ", err) + t.Log("WriteFrame (HEADERS): ", err) + return } if err := writer.WriteFrame(&synStreamFrame); err != nil { - t.Fatal("WriteFrame (SYN_STREAM): ", err) + t.Log("WriteFrame (SYN_STREAM): ", err) + return } }() diff --git a/bfe_tls/common.go b/bfe_tls/common.go index 8bf38041c..039aa64eb 100644 --- a/bfe_tls/common.go +++ b/bfe_tls/common.go @@ -224,6 +224,8 @@ type ConnectionState struct { ClientCiphers []uint16 // ciphers supported by client ClientAuth bool // enable TLS Client Authentication ClientCAName string // TLS client CA name + JA3Raw string // JA3 fingerprint string for TLS Client + JA3Hash string // JA3 fingerprint hash for TLS Client } // ClientAuthType declares the policy the server will follow for diff --git a/bfe_tls/conn.go b/bfe_tls/conn.go index 4948fe0b5..e9c756d20 100644 --- a/bfe_tls/conn.go +++ b/bfe_tls/conn.go @@ -76,6 +76,8 @@ type Conn struct { serverRandom []byte // random in server hello msg masterSecret []byte // master secret for conn clientCiphers []uint16 // ciphers supported by client + ja3Raw string // JA3 fingerprint string for TLS Client + ja3Hash string // JA3 fingerprint hash for TLS Client clientProtocol string clientProtocolFallback bool @@ -1276,6 +1278,8 @@ func (c *Conn) ConnectionState() ConnectionState { state.ClientAuth = true } state.ClientCAName = c.clientCAName + state.JA3Raw = c.ja3Raw + state.JA3Hash = c.ja3Hash } return state diff --git a/bfe_tls/handshake_messages.go b/bfe_tls/handshake_messages.go index 37db09f05..fb7e1ab23 100644 --- a/bfe_tls/handshake_messages.go +++ b/bfe_tls/handshake_messages.go @@ -20,6 +20,7 @@ package bfe_tls import ( "bytes" + "fmt" ) type clientHelloMsg struct { @@ -40,6 +41,7 @@ type clientHelloMsg struct { secureRenegotiation bool alpnProtocols []string padding bool + extensionIds []uint16 } func (m *clientHelloMsg) equal(i interface{}) bool { @@ -66,6 +68,45 @@ func (m *clientHelloMsg) equal(i interface{}) bool { eqStrings(m.alpnProtocols, m1.alpnProtocols) } +// JA3String returns a JA3 fingerprint string for TLS client. +// For more information, see https://github.com/salesforce/ja3 +func (m *clientHelloMsg) JA3String() string { + var buf bytes.Buffer + // version + fmt.Fprintf(&buf, "%d,", m.vers) + // cipher surites + writeJA3Uint16Values(&buf, m.cipherSuites) + fmt.Fprintf(&buf, ",") + // extensions + writeJA3Uint16Values(&buf, m.extensionIds) + fmt.Fprintf(&buf, ",") + // elliptic curves + for i, curve := range m.supportedCurves { + fmt.Fprintf(&buf, "%d", curve) + if i != len(m.supportedCurves)-1 { + fmt.Fprintf(&buf, "-") + } + } + fmt.Fprintf(&buf, ",") + // elliptic curves point formats + for i, point := range m.supportedPoints { + fmt.Fprintf(&buf, "%d", point) + if i != len(m.supportedPoints)-1 { + fmt.Fprintf(&buf, "-") + } + } + return buf.String() +} + +func writeJA3Uint16Values(buf *bytes.Buffer, values []uint16) { + for i, value := range values { + fmt.Fprintf(buf, "%d", value) + if i != len(values)-1 { + fmt.Fprintf(buf, "-") + } + } +} + func (m *clientHelloMsg) marshal() []byte { if m.raw != nil { return m.raw @@ -344,6 +385,7 @@ func (m *clientHelloMsg) unmarshal(data []byte) bool { m.signatureAndHashes = nil m.alpnProtocols = nil + m.extensionIds = make([]uint16, 0) if len(data) == 0 { // ClientHello is optionally followed by extension data return true @@ -369,6 +411,7 @@ func (m *clientHelloMsg) unmarshal(data []byte) bool { return false } + m.extensionIds = append(m.extensionIds, extension) switch extension { case extensionServerName: if length < 2 { diff --git a/bfe_tls/handshake_messages_test.go b/bfe_tls/handshake_messages_test.go index edd846d1f..04eb208fb 100644 --- a/bfe_tls/handshake_messages_test.go +++ b/bfe_tls/handshake_messages_test.go @@ -19,6 +19,8 @@ package bfe_tls import ( + "crypto/md5" + "fmt" "math/rand" "reflect" "testing" @@ -263,3 +265,34 @@ func (*sessionState) Generate(rand *rand.Rand, size int) reflect.Value { } return reflect.ValueOf(s) } + +var ja3HashTests = []struct { + vers uint16 + cipherSuites []uint16 + extensionIds []uint16 + supportedCurves []CurveID + supportedPoints []uint8 + ja3Hash string +}{ + {769, []uint16{47, 53, 5, 10, 49161, 49162, 49171, 49172, 50, 56, 19, 4}, + []uint16{0, 10, 11}, []CurveID{23, 24, 25}, []uint8{0}, + "ada70206e40642a3e4461f35503241d5"}, + {769, []uint16{4, 5, 10, 9, 100, 98, 3, 6, 19, 18, 99}, + []uint16{}, []CurveID{}, []uint8{}, + "de350869b8c85de67a350c8d186f11e6"}, +} + +func TestJA3Hash(t *testing.T) { + for i, d := range ja3HashTests { + msg := clientHelloMsg{} + msg.vers = d.vers + msg.cipherSuites = d.cipherSuites + msg.extensionIds = d.extensionIds + msg.supportedCurves = d.supportedCurves + msg.supportedPoints = d.supportedPoints + ja3Value := md5.Sum([]byte(msg.JA3String())) + if d.ja3Hash != fmt.Sprintf("%x", ja3Value) { + t.Errorf("#%d: unexpected ja3 value", i) + } + } +} diff --git a/bfe_tls/handshake_server.go b/bfe_tls/handshake_server.go index ef0730767..4118c4489 100644 --- a/bfe_tls/handshake_server.go +++ b/bfe_tls/handshake_server.go @@ -21,6 +21,7 @@ package bfe_tls import ( "crypto" "crypto/ecdsa" + "crypto/md5" "crypto/rsa" "crypto/subtle" "crypto/x509" @@ -73,6 +74,10 @@ func (c *Conn) serverHandshake() error { return err } + // Record JA3 fingerpint for TLS client + c.ja3Raw = hs.clientHello.JA3String() + c.ja3Hash = fmt.Sprintf("%x", md5.Sum([]byte(c.ja3Raw))) + // For an overview of TLS handshaking, see https://tools.ietf.org/html/rfc5246#section-7.3 if isResume { state.TlsHandshakeResumeAll.Inc(1) diff --git a/bfe_util/pipe/pipe.go b/bfe_util/pipe/pipe.go index 661ff1f7a..56919fa36 100644 --- a/bfe_util/pipe/pipe.go +++ b/bfe_util/pipe/pipe.go @@ -119,6 +119,12 @@ func (p *Pipe) closeWithError(dst *error, err error, fn func()) { } defer p.c.Signal() if *dst != nil { + // Note: Here we do not consider the existing io.EOF(i.e. *dst) as a real error + // and replace it if necessary. The error handling policy allows us to release + // underlying resource(eg. PipeBuffer) as soon as possible. + if *dst == io.EOF { + *dst = err + } // Already been done. return } diff --git a/docs/en_us/DOWNLOAD.md b/docs/en_us/DOWNLOAD.md index b73e7fdb7..fdcc7c93f 100644 --- a/docs/en_us/DOWNLOAD.md +++ b/docs/en_us/DOWNLOAD.md @@ -1,5 +1,17 @@ We provide precompiled binaries for bfe components. [Download the latest release](https://github.com/bfenetworks/bfe/releases) of BFE for your platform. +## bfe v1.0.0 + +* 2021-01-15 [Release notes](https://github.com/bfenetworks/bfe/releases/tag/v1.0.0) + +| File name | OS | Arch | Size | SHA256 Checksum | +| --------- | -- | ---- | ---- | --------------- | +| [bfe_1.0.0_darwin_amd64.tar.gz](https://github.com/bfenetworks/bfe/releases/download/v1.0.0/bfe_1.0.0_darwin_amd64.tar.gz) | darwin | amd64 | 7.03 MB | c0d13440d89ab97f52c61610d1b10dec6dcfb47b468a66078d1dd60f0541ec9e | +| [bfe_1.0.0_linux_arm64.tar.gz](https://github.com/bfenetworks/bfe/releases/download/v1.0.0/bfe_1.0.0_linux_arm64.tar.gz) | linux | arm64 | 5.63 MB | 47a3730ac90c4700c557d6c5903361c557e169102256bac870cede4eb90ff829 | +| [bfe_1.0.0_linux_amd64.tar.gz](https://github.com/bfenetworks/bfe/releases/download/v1.0.0/bfe_1.0.0_linux_amd64.tar.gz) | linux | amd64 | 6.18 MB | 5ec46c26827d554ba4c76f7f5e12b6b6afb68a9333213065802fa425fb81cbd1 | +| [bfe_1.0.0_windows_amd64.tar.gz](https://github.com/bfenetworks/bfe/releases/download/v1.0.0/bfe_1.0.0_windows_amd64.tar.gz) | windows | amd64 | 6.15 MB | 95ba788d0335ac536036c77e39249ce1629b2d159c942293077fd57ddc487f29 | + + ## bfe v0.10.0 * 2020-05-25 [Release notes](https://github.com/bfenetworks/bfe/releases/tag/v0.10.0) diff --git a/docs/en_us/condition/condition_primitive_index.md b/docs/en_us/condition/condition_primitive_index.md index 6fbe76a5e..9d1179e81 100644 --- a/docs/en_us/condition/condition_primitive_index.md +++ b/docs/en_us/condition/condition_primitive_index.md @@ -21,6 +21,7 @@ * req_proto_secure() * req_tag_match(tagName, tagValue) * req_path_in(path_list, case_insensitive) + * req_path_contain(path_list, case_insensitive) * req_path_prefix_in(prefix_list, case_insensitive) * req_path_element_prefix_in(prefix_list, case_insensitive) * req_path_suffix_in(suffix_list, case_insensitive) diff --git a/docs/en_us/condition/request/uri.md b/docs/en_us/condition/request/uri.md index 1b302dd1f..82206ab6f 100644 --- a/docs/en_us/condition/request/uri.md +++ b/docs/en_us/condition/request/uri.md @@ -35,6 +35,22 @@ req_host_in("www.bfe-networks.com | bfe-networks.com") req_path_in("/api/search|/api/list", true) ``` +## req_path_contain(path_list, case_insensitive) +* Description: Judge if request path contains configured patterns + +* Parameters + +| Parameter | Descrption | +| --------- | ---------- | +| path_list | String
path's substring list which are concatenated with | | +| case_insensitive | Boolean
case insensitive | + +* Example + +```go +req_path_contain("search|analytics", true) +``` + ## req_path_prefix_in(prefix_list, case_insensitive) * Description: Judge if request path prefix matches configured patterns diff --git a/docs/en_us/configuration/server_data_conf/cluster_conf.data.md b/docs/en_us/configuration/server_data_conf/cluster_conf.data.md index a293045f2..e7e02a1ad 100644 --- a/docs/en_us/configuration/server_data_conf/cluster_conf.data.md +++ b/docs/en_us/configuration/server_data_conf/cluster_conf.data.md @@ -23,7 +23,9 @@ BackendConf is config for backend. | TimeoutConnSrv | Int
Timeout for connect backend, in ms | | TimeoutResponseHeader | Int
Timeout for read response header, in ms | | MaxIdleConnsPerHost | Int
Max idle conns to each backend | +| MaxConnsPerHost | Int
Max number of concurrent conns to each backend | | RetryLevel | Int
Retry level if request fail | +| BackendConf.OutlierDetectionHttpCode | String
Http status code that represent error status of backend | | FCGIConf | Object
Conf for FastCGI Protocol | | FCGIConf.Root | String
the root folder to the site | | FCGIConf.EnvVars | Map[string]string
extra environment variable | @@ -78,7 +80,9 @@ ClusterBasic is basic config for cluster. "TimeoutConnSrv": 2000, "TimeoutResponseHeader": 50000, "MaxIdleConnsPerHost": 0, - "RetryLevel": 0 + "MaxConnsPerHost": 0, + "RetryLevel": 0, + "OutlierDetectionHttpCode": "5xx|400" }, "CheckConf": { "Schem": "http", diff --git a/docs/en_us/modules/mod_header/mod_header.md b/docs/en_us/modules/mod_header/mod_header.md index 34c258743..5bed26f2e 100644 --- a/docs/en_us/modules/mod_header/mod_header.md +++ b/docs/en_us/modules/mod_header/mod_header.md @@ -108,6 +108,8 @@ See the **Example** above. | %bfe_ssl_resume | Whether the TLS/SSL session is resumed with session id or session ticket | | %bfe_ssl_cipher | TLS/SSL cipher suite | | %bfe_ssl_version | TLS/SSL version | +| %bfe_ssl_ja3_raw | JA3 fingerprint string for TLS/SSL client | +| %bfe_ssl_ja3_hash | JA3 fingerprint hash for TLS/SSL client | | %bfe_protocol | Application level protocol | | %client_cert_serial_number | Serial number of client certificate | | %client_cert_subject_title | Subject title of client certificate | diff --git a/docs/mkdocs_en.yml b/docs/mkdocs_en.yml index 0abc0ab86..1787b46a1 100644 --- a/docs/mkdocs_en.yml +++ b/docs/mkdocs_en.yml @@ -69,12 +69,14 @@ nav: - 'Request redirect': 'example/redirect.md' - 'Request rewrite': 'example/rewrite.md' - 'TLS mutual authentication': 'example/client_auth.md' + - 'FastCGI': 'example/fastcgi.md' - 'Installation': - 'Overview': 'installation/install.md' - 'Install from source': 'installation/install_from_source.md' - 'Install using binaries': 'installation/install_using_binaries.md' - 'Install using go': 'installation/install_using_go.md' - 'Install using snap': 'installation/install_using_snap.md' + - 'Install using docker': 'installation/install_using_docker.md' - 'Configuration': - 'Overview': 'configuration/config.md' - 'Core': 'configuration/bfe.conf.md' diff --git a/docs/mkdocs_zh.yml b/docs/mkdocs_zh.yml index 43304ee9b..0f046c23d 100644 --- a/docs/mkdocs_zh.yml +++ b/docs/mkdocs_zh.yml @@ -69,12 +69,14 @@ nav: - '重定向': 'example/redirect.md' - '重写': 'example/rewrite.md' - 'TLS客户端认证': 'example/client_auth.md' + - 'FastCGI': 'example/fastcgi.md' - '安装说明': - '安装概述': 'installation/install.md' - '源码编译安装': 'installation/install_from_source.md' - '二进制文件下载安装': 'installation/install_using_binaries.md' - 'go方式安装': 'installation/install_using_go.md' - 'snap方式安装': 'installation/install_using_snap.md' + - 'docker方式安装': 'installation/install_using_docker.md' - '配置说明': - '配置概述': 'configuration/config.md' - '核心配置': 'configuration/bfe.conf.md' diff --git a/docs/zh_cn/DOWNLOAD.md b/docs/zh_cn/DOWNLOAD.md index f6d2e7a67..cab6a53e7 100644 --- a/docs/zh_cn/DOWNLOAD.md +++ b/docs/zh_cn/DOWNLOAD.md @@ -1,5 +1,16 @@ BFE提供预编译二进制文件供下载。也可在GitHub下载各平台[最新版本BFE](https://github.com/bfenetworks/bfe/releases)。 +## bfe v1.0.0 + +* 2021-01-15 [发布说明](https://github.com/bfenetworks/bfe/releases/tag/v1.0.0) + +| 文件名 | 操作系统 | 平台 | 大小 | SHA256检验和 | +| --------- | -- | ---- | ---- | --------------- | +| [bfe_1.0.0_darwin_amd64.tar.gz](https://github.com/bfenetworks/bfe/releases/download/v1.0.0/bfe_1.0.0_darwin_amd64.tar.gz) | darwin | amd64 | 7.03 MB | c0d13440d89ab97f52c61610d1b10dec6dcfb47b468a66078d1dd60f0541ec9e | +| [bfe_1.0.0_linux_arm64.tar.gz](https://github.com/bfenetworks/bfe/releases/download/v1.0.0/bfe_1.0.0_linux_arm64.tar.gz) | linux | arm64 | 5.63 MB | 47a3730ac90c4700c557d6c5903361c557e169102256bac870cede4eb90ff829 | +| [bfe_1.0.0_linux_amd64.tar.gz](https://github.com/bfenetworks/bfe/releases/download/v1.0.0/bfe_1.0.0_linux_amd64.tar.gz) | linux | amd64 | 6.18 MB | 5ec46c26827d554ba4c76f7f5e12b6b6afb68a9333213065802fa425fb81cbd1 | +| [bfe_1.0.0_windows_amd64.tar.gz](https://github.com/bfenetworks/bfe/releases/download/v1.0.0/bfe_1.0.0_windows_amd64.tar.gz) | windows | amd64 | 6.15 MB | 95ba788d0335ac536036c77e39249ce1629b2d159c942293077fd57ddc487f29 | + ## bfe v0.10.0 * 2020-05-25 [发布说明](https://github.com/bfenetworks/bfe/releases/tag/v0.10.0) diff --git a/docs/zh_cn/condition/condition_primitive_index.md b/docs/zh_cn/condition/condition_primitive_index.md index fe0de0183..751061ddf 100644 --- a/docs/zh_cn/condition/condition_primitive_index.md +++ b/docs/zh_cn/condition/condition_primitive_index.md @@ -21,6 +21,7 @@ * req_proto_secure() * req_tag_match(tagName, tagValue) * req_path_in(path_list, case_insensitive) + * req_path_contain(path_list, case_insensitive) * req_path_prefix_in(prefix_list, case_insensitive) * req_path_suffix_in(suffix_list, case_insensitive) * req_path_element_suffix_in(suffix_list, case_insensitive) diff --git a/docs/zh_cn/condition/request/uri.md b/docs/zh_cn/condition/request/uri.md index 2496cf68c..8d1f42c19 100644 --- a/docs/zh_cn/condition/request/uri.md +++ b/docs/zh_cn/condition/request/uri.md @@ -28,6 +28,21 @@ req_host_in("www.bfe-networks.com|bfe-networks.com") req_path_in("/api/search|/api/list", true) ``` +## req_path_contain(path_list, case_insensitive) +* 含义： 判断http的path是否包含path_list中的子串 + +* 参数 + +| 参数 | 描述 | +| -------- | ---------------------- | +| path_list | String
path子串列表，多个列表之间使用‘|’连接| +| case_insensitive | Boolean
是否忽略大小写 | + +* 示例 +```go +req_path_contain("search", true) +``` + ## req_path_prefix_in(prefix_list, case_insensitive) * 含义： 判断http的path是否前缀匹配prefix_list之一 diff --git a/docs/zh_cn/configuration/server_data_conf/cluster_conf.data.md b/docs/zh_cn/configuration/server_data_conf/cluster_conf.data.md index 30eaa4866..b88856ff6 100644 --- a/docs/zh_cn/configuration/server_data_conf/cluster_conf.data.md +++ b/docs/zh_cn/configuration/server_data_conf/cluster_conf.data.md @@ -27,7 +27,9 @@ cluster_conf.data为集群转发配置文件。 | BackendConf.TimeoutConnSrv | Integer
连接后端的超时时间，单位是毫秒
默认值2 | | BackendConf.TimeoutResponseHeader | Integer
从后端读响应头的超时时间，单位是毫秒
默认值60 | | BackendConf.MaxIdleConnsPerHost | Integer
BFE实例与每个后端的最大空闲长连接数
默认值2 | +| BackendConf.MaxConnsPerHost | Integer
BFE实例与每个后端的最大长连接数，0代表无限制
默认值0 | | BackendConf.RetryLevel | Integer
请求重试级别。0：连接后端失败时，进行重试；1：连接后端失败、转发GET请求失败时均进行重试
默认值0 | +| BackendConf.OutlierDetectionHttpCode | String
后端响应状态码检查，""代表不开启检查，"500"表示后端返回500则认为后端失败，失败计数加一
状态码支持"dxx"格式，例如"5xx";多个状态码之间使用'|'连接
默认值""，不开启后端响应状态码错误检查 | | BackendConf.FCGIConf | Object
FastCGI 协议的配置 | | BackendConf.FCGIConf.Root | String
网站的Root文件夹位置 | | BackendConf.FCGIConf.EnvVars | Map[string]string
拓展的环境变量 | @@ -59,11 +61,11 @@ cluster_conf.data为集群转发配置文件。 #### 集群基础配置 -| 配置项 | 描述 | -| ----------------------------------- | ------------------------------------ | -| ClusterBasic.TimeoutReadClient | Integer
读用户请求wody的超时时间，单位为毫秒
默认值30 | -| ClusterBasic.TimeoutWriteClient | Integer
写响应的超时时间，单位为毫秒
默认值60 | -| ClusterBasic.TimeoutReadClientAgain | Integer
连接闲置超时时间，单位为毫秒
默认值60 | +| 配置项 | 描述 | +| ----------------------------------- | ----------------------------------------------------------- | +| ClusterBasic.TimeoutReadClient | Integer
读用户请求body的超时时间，单位为毫秒
默认值30 | +| ClusterBasic.TimeoutWriteClient | Integer
写响应的超时时间，单位为毫秒
默认值60 | +| ClusterBasic.TimeoutReadClientAgain | Integer
连接闲置超时时间，单位为毫秒
默认值60 | ## 配置示例 @@ -76,7 +78,8 @@ cluster_conf.data为集群转发配置文件。 "TimeoutConnSrv": 2000, "TimeoutResponseHeader": 50000, "MaxIdleConnsPerHost": 0, - "RetryLevel": 0 + "RetryLevel": 0, + "OutlierDetectionHttpCode": "5xx|403" }, "CheckConf": { "Schem": "http", @@ -107,6 +110,7 @@ cluster_conf.data为集群转发配置文件。 "TimeoutConnSrv": 2000, "TimeoutResponseHeader": 50000, "MaxIdleConnsPerHost": 0, + "MaxConnsPerHost": 0, "RetryLevel": 0, "FCGIConf": { "Root": "/home/work", diff --git a/docs/zh_cn/development/local_dev_guide.md b/docs/zh_cn/development/local_dev_guide.md index 35b09b2da..4337e0645 100644 --- a/docs/zh_cn/development/local_dev_guide.md +++ b/docs/zh_cn/development/local_dev_guide.md @@ -116,7 +116,7 @@ clang-formater.......................................(no files to check)Skipped # 触发develop分支的CI单测 $ git commit -m "test=develop" -# 触发release/1.1分支的CI单侧 +# 触发release/1.1分支的CI单测 $ git commit -m "test=release/1.1" ``` diff --git a/docs/zh_cn/modules/mod_header/mod_header.md b/docs/zh_cn/modules/mod_header/mod_header.md index c365f3217..b262954af 100644 --- a/docs/zh_cn/modules/mod_header/mod_header.md +++ b/docs/zh_cn/modules/mod_header/mod_header.md @@ -102,6 +102,8 @@ BFE支持如下一系列变量并在处理请求阶段求值。关于变量的 | %bfe_ssl_resume | 是否TLS/SSL会话复用 | | %bfe_ssl_cipher | TLS/SSL加密套件 | | %bfe_ssl_version | TLS/SSL协议版本 | +| %bfe_ssl_ja3_raw | TLS/SSL客户端JA3算法指纹数据 | +| %bfe_ssl_ja3_hash | TLS/SSL客户端JA3算法指纹哈希值 | | %bfe_protocol | 访问协议 | | %client_cert_serial_number | 客户端证书序列号 | | %client_cert_subject_title | 客户端证书Subject title | diff --git a/go.mod b/go.mod index 91e651001..a5612e485 100644 --- a/go.mod +++ b/go.mod @@ -29,9 +29,9 @@ require ( go.elastic.co/apm/module/apmot v1.7.2 go.uber.org/atomic v1.6.0 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/net v0.0.0-20200625001655-4c5254603344 - golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd - golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7 // indirect + golang.org/x/net v0.0.0-20201021035429-f5854403a974 + golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 + golang.org/x/tools v0.1.0 // indirect gopkg.in/gcfg.v1 v1.2.3 gopkg.in/square/go-jose.v2 v2.4.1 gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 862b3a750..171873994 100644 --- a/go.sum +++ b/go.sum @@ -124,7 +124,7 @@ github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR8 github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zmap/go-iptree v0.0.0-20170831022036-1948b1097e25 h1:LRoXAcKX48QV4LV23W5ZtsG/MbJOgNUNvWiXwM0iLWw= github.com/zmap/go-iptree v0.0.0-20170831022036-1948b1097e25/go.mod h1:qOasALtPByO1Jk6LhgpNv6htPMK2QJfiGorUk57nO/U= go.elastic.co/apm v1.7.2 h1:0nwzVIPp4PDBXSYYtN19+1W5V+sj+C25UjqxDVoKcA8= @@ -155,15 +155,15 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -171,26 +171,28 @@ golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191018212557-ed542cd5b28a h1:UuQ+70Pi/ZdWHuP4v457pkXeOynTdgd/4enxeIO/98k= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7 h1:LHW24ah7B+uV/OePwNP0p/t889F3QSyLvY8Sg/bK0SY= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=