Skip to content

Commit

Permalink
Optimize dynamic detect logic
Browse files Browse the repository at this point in the history
  • Loading branch information
wweir committed Feb 18, 2020
1 parent c38d519 commit 5d1827b
Show file tree
Hide file tree
Showing 7 changed files with 74 additions and 172 deletions.
23 changes: 9 additions & 14 deletions conf/conf.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package conf
import (
"flag"
"os"
"sync"
"time"

toml "github.com/pelletier/go-toml"
Expand All @@ -29,9 +28,9 @@ type client struct {
DetectLevel int `toml:"detect_level"`
DetectTimeout string `toml:"detect_timeout"`

ProxyList []string `toml:"proxy_list"`
DirectList []string `toml:"direct_list"`
DynamicList []string `toml:"dynamic_list"`
ProxyList []string `toml:"proxy_list"`
DirectList []string `toml:"direct_list"`
DynamicList map[string]int `toml:"dynamic_list"` // toml has a bug with dot
directRules *util.Node
proxyRules *util.Node
dynamicRules *util.Node
Expand All @@ -47,17 +46,14 @@ type server struct {
var (
version, date string

flushOnce = sync.Once{}
flushMu = sync.Mutex{}
flushCh = make(chan struct{})

Server = server{}
Client = client{}
conf = struct {
file string
Server *server `toml:"server"`
Client *client `toml:"client"`
}{"", &Server, &Client}
flushCh = make(chan struct{})
Password string
installCmd string
uninstallFlag bool
Expand Down Expand Up @@ -108,6 +104,8 @@ func init() {
log.Fatalw("load config", "config", conf.file, "step", loadConfigFns[i].step, "err", err)
}
}

go flushConfDaemon()
}

// refreshFns will be executed while init and write new config
Expand All @@ -121,12 +119,12 @@ var loadConfigFns = []struct {
}
defer f.Close()

Client.Router.DynamicList = map[string]int{}
return toml.NewDecoder(f).Decode(&conf)

}}, {"load_rules", func() error {
Client.Router.directRules = util.NewNodeFromRules(Client.Router.DirectList...)
Client.Router.proxyRules = util.NewNodeFromRules(Client.Router.ProxyList...)
Client.Router.dynamicRules = util.NewNodeFromRules(Client.Router.DynamicList...)
return nil

}}, {"flush_dns", func() error {
Expand All @@ -136,24 +134,21 @@ var loadConfigFns = []struct {
return nil
}}}

func flushConf() {
func flushConfDaemon() {
for range flushCh {
// safe write
// safe write file
if conf.file != "" {
f, err := os.OpenFile(conf.file+"~", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
log.Errorw("flush config", "step", "flush", "err", err)
continue
}

flushMu.Lock()
if err := toml.NewEncoder(f).ArraysWithOneElementPerLine(true).Encode(conf); err != nil {
log.Errorw("flush config", "step", "flush", "err", err)
flushMu.Unlock()
f.Close()
continue
}
flushMu.Unlock()
f.Close()

if err = os.Rename(conf.file+"~", conf.file); err != nil {
Expand Down
100 changes: 61 additions & 39 deletions conf/dynamic_rule.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (

"github.com/wweir/sower/internal/http"
"github.com/wweir/sower/internal/socks5"
"github.com/wweir/sower/util"
"github.com/wweir/utils/log"
"github.com/wweir/utils/mem"
)
Expand All @@ -19,10 +18,12 @@ type dynamic struct {
port http.Port
}

var cache = mem.New(2 * time.Hour)
var cache = mem.New(4 * time.Hour)
var detect = &dynamic{}
var passwordData []byte
var timeout time.Duration
var dynamicCache sync.Map
var dynamicMu = sync.Mutex{}

// ShouldProxy check if the domain shoule request though proxy
func ShouldProxy(domain string) bool {
Expand All @@ -35,12 +36,10 @@ func ShouldProxy(domain string) bool {
if Client.Router.proxyRules.Match(domain) {
return true
}
if Client.Router.dynamicRules.Match(domain) {
return true
}

cache.Remember(detect, domain)
return Client.Router.dynamicRules.Match(domain)
val, _ := dynamicCache.Load(domain)
return val.(int) >= Client.Router.DetectLevel
}

func (d *dynamic) Get(key interface{}) (err error) {
Expand All @@ -49,7 +48,53 @@ func (d *dynamic) Get(key interface{}) (err error) {
if strings.Count(domain, ".") > 10 {
return nil
}
domainUnderscore := strings.ReplaceAll(domain, ".", "_")
var score int

defer func() {
dynamicCache.Store(domain, score)

if score < conf.Client.Router.DetectLevel {
delete(Client.Router.DynamicList, domainUnderscore)
} else {
Client.Router.DynamicList[domainUnderscore] = score

// persist when add new domain
select {
case flushCh <- struct{}{}:
default:
}
log.Infow("persist rule", "domain", domain, "score", score)
}
}()

if val, ok := dynamicCache.Load(domain); ok {
score = val.(int)
} else {
dynamicMu.Lock()
score = Client.Router.DynamicList[domainUnderscore]
dynamicMu.Unlock()
}

// detect range: [0,conf.Client.Router.DetectLevel)
switch {
case score < -1:
score++
case score == -1:
score++
score += d.detect(domain)
case score > conf.Client.Router.DetectLevel:
score--
case score == conf.Client.Router.DetectLevel:
score--
score += d.detect(domain)
}

return nil
}

// detect and caculate direct connection and proxy connection score
func (d *dynamic) detect(domain string) int {
wg := sync.WaitGroup{}
httpScore, httpsScore := new(int32), new(int32)
for _, ping := range [...]dynamic{{port: http.HTTP}, {port: http.HTTPS}} {
Expand All @@ -63,12 +108,12 @@ func (d *dynamic) Get(key interface{}) (err error) {

switch ping.port {
case http.HTTP:
if !atomic.CompareAndSwapInt32(httpScore, 0, 2) {
atomic.AddInt32(httpScore, 1)
if !atomic.CompareAndSwapInt32(httpScore, 0, -2) {
atomic.AddInt32(httpScore, -1)
}
case http.HTTPS:
if !atomic.CompareAndSwapInt32(httpsScore, 0, 2) {
atomic.AddInt32(httpScore, 1)
if !atomic.CompareAndSwapInt32(httpsScore, 0, -2) {
atomic.AddInt32(httpScore, -1)
}
}
}(ping)
Expand All @@ -79,6 +124,7 @@ func (d *dynamic) Get(key interface{}) (err error) {
defer wg.Done()

var conn net.Conn
var err error
if addr, ok := socks5.IsSocks5Schema(Client.Address); ok {
conn, err = net.Dial("tcp", addr)
conn = socks5.ToSocks5(conn, domain, uint16(ping.port))
Expand All @@ -102,41 +148,17 @@ func (d *dynamic) Get(key interface{}) (err error) {

switch ping.port {
case http.HTTP:
if !atomic.CompareAndSwapInt32(httpScore, 0, -2) {
atomic.AddInt32(httpScore, -1)
if !atomic.CompareAndSwapInt32(httpScore, 0, 2) {
atomic.AddInt32(httpScore, 1)
}
case http.HTTPS:
if !atomic.CompareAndSwapInt32(httpsScore, 0, -2) {
atomic.AddInt32(httpScore, -1)
if !atomic.CompareAndSwapInt32(httpsScore, 0, 2) {
atomic.AddInt32(httpScore, 1)
}
}
}(ping)
}

wg.Wait()
if int(*httpScore+*httpsScore)+conf.Client.Router.DetectLevel < 0 {
addDynamic(domain)
log.Infow("add rule", "domain", domain, "http_score", *httpScore, "https_score", *httpsScore)
}
return nil
}

// addDynamic add new domain into dynamic list
func addDynamic(domain string) {
flushMu.Lock()
Client.Router.DynamicList = util.NewReverseSecSlice(
append(Client.Router.DynamicList, domain)).Sort().Uniq()
Client.Router.dynamicRules = util.NewNodeFromRules(Client.Router.DynamicList...)
flushMu.Unlock()

flushOnce.Do(func() {
if conf.file != "" {
go flushConf()
}
})

select {
case flushCh <- struct{}{}:
default:
}
return int(*httpScore + *httpsScore)
}
2 changes: 1 addition & 1 deletion conf/sower.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
"pop.*.*.*",
"**.cn",
]
dynamic_list = []
proxy_list = [
"**.google.*",
"**.goo.gl",
Expand All @@ -44,6 +43,7 @@
"**.amazon.com",
"**.amazonaws.com",
"*.githubusercontent.com",
"*.githubassets.com",
"*.github.*",
]

Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
Expand Down Expand Up @@ -88,6 +89,7 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnf
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
Expand Down
2 changes: 1 addition & 1 deletion proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func StartServer(relayTarget, password, certFile, keyFile, email string) {

rc, err := net.Dial("tcp", addr)
if err != nil {
log.Errorw("tcp dial", "host", domain, "addr", addr, "err", err)
log.Errorw("tcp dial", "addr", addr, "err", err)
return
}
defer rc.Close()
Expand Down
52 changes: 0 additions & 52 deletions util/rev_sec.go

This file was deleted.

Loading

0 comments on commit 5d1827b

Please sign in to comment.