From 84761ce16abd27c2a2003715fe63179cc8de4555 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Wed, 4 Aug 2021 02:43:35 +0000 Subject: [PATCH 01/14] feat: new api for NewLogger --- logger.go | 147 +++++++++++++++++++++++++++++++++++++++---------- logger_test.go | 2 +- 2 files changed, 120 insertions(+), 29 deletions(-) diff --git a/logger.go b/logger.go index f56da2b..60a4564 100644 --- a/logger.go +++ b/logger.go @@ -75,50 +75,141 @@ func CreateNewDefaultLogger(name string, level LoggerLevel, opts ...zap.Option) return Logger, nil } -// NewLogger create new logger -func NewLogger(level LoggerLevel, opts ...zap.Option) (l *LoggerType, err error) { - return NewLoggerWithName("", level, opts...) -} - // NewLoggerWithName create new logger with name func NewLoggerWithName(name string, level LoggerLevel, opts ...zap.Option) (l *LoggerType, err error) { - return NewLoggerWithNameAndFormat(name, "json", level, opts...) + return NewLogger( + WithLoggerName(name), + WithLoggerEncoding(LoggerEncodingJSON), + WithLoggerLevel(level), + WithLoggerZapOptions(opts...), + ) } // NewConsoleLoggerWithName create new logger with name func NewConsoleLoggerWithName(name string, level LoggerLevel, opts ...zap.Option) (l *LoggerType, err error) { - return NewLoggerWithNameAndFormat(name, "console", level, opts...) + return NewLogger( + WithLoggerName(name), + WithLoggerEncoding(LoggerEncodingConsole), + WithLoggerLevel(level), + WithLoggerZapOptions(opts...), + ) +} + +type LoggerConfig struct { + zap.Config + zapOptions []zap.Option + Name string } -// NewLoggerWithNameAndFormat create new logger -func NewLoggerWithNameAndFormat(name, format string, level LoggerLevel, opts ...zap.Option) (l *LoggerType, err error) { - zl := zap.NewAtomicLevel() - cfg := zap.Config{ - Level: zl, - Development: false, - Encoding: format, - EncoderConfig: zap.NewProductionEncoderConfig(), - OutputPaths: []string{"stdout"}, - ErrorOutputPaths: []string{"stderr"}, +type LoggerEncoding string + +const ( + LoggerEncodingConsole = "console" + LoggerEncodingJSON = "json" +) + +type LoggerOption func(l *LoggerConfig) error + +func WithLoggerOutputFiles(files []string) LoggerOption { + return func(c *LoggerConfig) error { + c.OutputPaths = append(c.OutputPaths, files...) + return nil } - cfg.EncoderConfig.EncodeCaller = zapcore.ShortCallerEncoder - cfg.EncoderConfig.MessageKey = "message" - cfg.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder - if format == "console" { - cfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder +} + +func WithLoggerErrorOutputFiles(files []string) LoggerOption { + return func(c *LoggerConfig) error { + c.ErrorOutputPaths = append(c.OutputPaths, files...) + return nil } +} + +func WithLoggerEncoding(format LoggerEncoding) LoggerOption { + return func(c *LoggerConfig) error { + switch format { + case LoggerEncodingConsole: + c.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + case LoggerEncodingJSON: + default: + return errors.Errorf("invalid format: %s", format) + } + + return nil + } +} - zapLogger, err := cfg.Build(opts...) +func WithLoggerZapOptions(opts ...zap.Option) LoggerOption { + return func(c *LoggerConfig) error { + c.zapOptions = opts + return nil + } +} + +func WithLoggerName(name string) LoggerOption { + return func(c *LoggerConfig) error { + c.Name = name + return nil + } +} + +func WithLoggerLevel(level LoggerLevel) LoggerOption { + return func(c *LoggerConfig) error { + switch level { + case LoggerLevelInfo: + c.Level.SetLevel(zap.InfoLevel) + case LoggerLevelDebug: + c.Level.SetLevel(zap.DebugLevel) + case LoggerLevelWarn: + c.Level.SetLevel(zap.WarnLevel) + case LoggerLevelError: + c.Level.SetLevel(zap.ErrorLevel) + case LoggerLevelFatal: + c.Level.SetLevel(zap.FatalLevel) + case LoggerLevelPanic: + c.Level.SetLevel(zap.PanicLevel) + default: + return errors.Errorf("invalid level: %s", level) + } + + return nil + } +} + +// NewLogger create new logger +func NewLogger(optfs ...LoggerOption) (l *LoggerType, err error) { + opt := &LoggerConfig{ + Name: "app", + Config: zap.Config{ + Level: zap.NewAtomicLevel(), + Development: false, + Encoding: string(LoggerEncodingConsole), + EncoderConfig: zap.NewProductionEncoderConfig(), + OutputPaths: []string{"stdout"}, + ErrorOutputPaths: []string{"stderr"}, + }, + } + opt.EncoderConfig.EncodeCaller = zapcore.ShortCallerEncoder + opt.EncoderConfig.MessageKey = "message" + opt.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder + + for _, optf := range optfs { + if err = optf(opt); err != nil { + return nil, errors.Wrap(err, "create logger") + } + } + + zapLogger, err := opt.Build(opt.zapOptions...) if err != nil { return nil, fmt.Errorf("build zap logger: %+v", err) } - zapLogger = zapLogger.Named(name) + zapLogger = zapLogger.Named(opt.Name) l = &LoggerType{ Logger: zapLogger, - level: zl, + level: opt.Level, } - return l, l.ChangeLevel(level) + + return l, nil } // Level get current level of logger @@ -142,10 +233,10 @@ func (l *LoggerType) ChangeLevel(level LoggerLevel) (err error) { case LoggerLevelFatal: l.level.SetLevel(zap.FatalLevel) default: - return fmt.Errorf("log level only be debug/info/warn/error") + return errors.Errorf("invalid level %s", string(level)) } - l.Debug("set lovel level", zap.String("level", level.String())) + l.Debug("set logger level", zap.String("level", level.String())) return } diff --git a/logger_test.go b/logger_test.go index 98ed658..895d1be 100644 --- a/logger_test.go +++ b/logger_test.go @@ -18,7 +18,7 @@ func TestNewLogger(t *testing.T) { lvl := logger.Level() require.Equal(t, LoggerLevelDebug, lvl) - _, err = NewLogger(LoggerLevelDebug) + _, err = NewLogger() require.NoError(t, err) logger = logger.Clone().Named("sample") From 1692c269594d3379709f06549eb5f84cec3c945e Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Wed, 4 Aug 2021 02:49:07 +0000 Subject: [PATCH 02/14] feat: change `WithLoggerErrorOutputPaths` --- logger.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/logger.go b/logger.go index 60a4564..d004304 100644 --- a/logger.go +++ b/logger.go @@ -110,16 +110,22 @@ const ( type LoggerOption func(l *LoggerConfig) error -func WithLoggerOutputFiles(files []string) LoggerOption { +// WithLoggerOutputPaths set output path +// +// like "stdout" +func WithLoggerOutputPaths(paths []string) LoggerOption { return func(c *LoggerConfig) error { - c.OutputPaths = append(c.OutputPaths, files...) + c.OutputPaths = paths return nil } } -func WithLoggerErrorOutputFiles(files []string) LoggerOption { +// WithLoggerErrorOutputPaths set error logs output path +// +// like "stderr" +func WithLoggerErrorOutputPaths(paths []string) LoggerOption { return func(c *LoggerConfig) error { - c.ErrorOutputPaths = append(c.OutputPaths, files...) + c.ErrorOutputPaths = paths return nil } } From 01e7b9a0da0a5cdf62ecfb91e9446aac57f97437 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Wed, 4 Aug 2021 07:04:20 +0000 Subject: [PATCH 03/14] feat: change logger level --- logger.go | 105 ++++++++++++++++++++++++++----------------------- logger_test.go | 52 +++++++++++++++++++++++- utils.go | 3 -- 3 files changed, 106 insertions(+), 54 deletions(-) diff --git a/logger.go b/logger.go index d004304..8bb03eb 100644 --- a/logger.go +++ b/logger.go @@ -16,12 +16,6 @@ import ( "github.com/pkg/errors" ) -type LoggerLevel string - -func (l LoggerLevel) String() string { - return string(l) -} - const ( // SampleRateDenominator sample rate = sample / SampleRateDenominator SampleRateDenominator = 1000 @@ -31,17 +25,17 @@ const ( defaultAlertHookLevel = zapcore.ErrorLevel // LoggerLevelInfo Logger level info - LoggerLevelInfo LoggerLevel = "info" + LoggerLevelInfo string = "info" // LoggerLevelDebug Logger level debug - LoggerLevelDebug LoggerLevel = "debug" + LoggerLevelDebug string = "debug" // LoggerLevelWarn Logger level warn - LoggerLevelWarn LoggerLevel = "warn" + LoggerLevelWarn string = "warn" // LoggerLevelError Logger level error - LoggerLevelError LoggerLevel = "error" + LoggerLevelError string = "error" // LoggerLevelFatal Logger level fatal - LoggerLevelFatal LoggerLevel = "fatal" + LoggerLevelFatal string = "fatal" // LoggerLevelPanic Logger level panic - LoggerLevelPanic LoggerLevel = "panic" + LoggerLevelPanic string = "panic" ) var ( @@ -62,11 +56,16 @@ var ( // LoggerType extend from zap.Logger type LoggerType struct { *zap.Logger + + // level level of current logger + // + // zap logger do not expose api to change log's level, + // so we have to save the pointer of zap.AtomicLevel. level zap.AtomicLevel } // CreateNewDefaultLogger set default utils.Logger -func CreateNewDefaultLogger(name string, level LoggerLevel, opts ...zap.Option) (l *LoggerType, err error) { +func CreateNewDefaultLogger(name, level string, opts ...zap.Option) (l *LoggerType, err error) { if l, err = NewLoggerWithName(name, level, opts...); err != nil { return nil, errors.Wrap(err, "create new logger") } @@ -76,7 +75,7 @@ func CreateNewDefaultLogger(name string, level LoggerLevel, opts ...zap.Option) } // NewLoggerWithName create new logger with name -func NewLoggerWithName(name string, level LoggerLevel, opts ...zap.Option) (l *LoggerType, err error) { +func NewLoggerWithName(name, level string, opts ...zap.Option) (l *LoggerType, err error) { return NewLogger( WithLoggerName(name), WithLoggerEncoding(LoggerEncodingJSON), @@ -86,7 +85,7 @@ func NewLoggerWithName(name string, level LoggerLevel, opts ...zap.Option) (l *L } // NewConsoleLoggerWithName create new logger with name -func NewConsoleLoggerWithName(name string, level LoggerLevel, opts ...zap.Option) (l *LoggerType, err error) { +func NewConsoleLoggerWithName(name, level string, opts ...zap.Option) (l *LoggerType, err error) { return NewLogger( WithLoggerName(name), WithLoggerEncoding(LoggerEncodingConsole), @@ -130,6 +129,7 @@ func WithLoggerErrorOutputPaths(paths []string) LoggerOption { } } +// WithLoggerEncoding set logger encoding formet func WithLoggerEncoding(format LoggerEncoding) LoggerOption { return func(c *LoggerConfig) error { switch format { @@ -144,6 +144,7 @@ func WithLoggerEncoding(format LoggerEncoding) LoggerOption { } } +// WithLoggerZapOptions set logger with zap.Option func WithLoggerZapOptions(opts ...zap.Option) LoggerOption { return func(c *LoggerConfig) error { c.zapOptions = opts @@ -151,6 +152,7 @@ func WithLoggerZapOptions(opts ...zap.Option) LoggerOption { } } +// WithLoggerName set logger name func WithLoggerName(name string) LoggerOption { return func(c *LoggerConfig) error { c.Name = name @@ -158,25 +160,35 @@ func WithLoggerName(name string) LoggerOption { } } -func WithLoggerLevel(level LoggerLevel) LoggerOption { +// ParseLoggerLevel +func ParseLoggerLevel(level string) (zapcore.Level, error) { + switch level { + case LoggerLevelInfo: + return zap.InfoLevel, nil + case LoggerLevelDebug: + return zap.DebugLevel, nil + case LoggerLevelWarn: + return zap.WarnLevel, nil + case LoggerLevelError: + return zap.ErrorLevel, nil + case LoggerLevelFatal: + return zap.FatalLevel, nil + case LoggerLevelPanic: + return zap.PanicLevel, nil + default: + return 0, errors.Errorf("invalid level: %s", level) + } +} + +// WithLoggerLevel set logger level +func WithLoggerLevel(level string) LoggerOption { return func(c *LoggerConfig) error { - switch level { - case LoggerLevelInfo: - c.Level.SetLevel(zap.InfoLevel) - case LoggerLevelDebug: - c.Level.SetLevel(zap.DebugLevel) - case LoggerLevelWarn: - c.Level.SetLevel(zap.WarnLevel) - case LoggerLevelError: - c.Level.SetLevel(zap.ErrorLevel) - case LoggerLevelFatal: - c.Level.SetLevel(zap.FatalLevel) - case LoggerLevelPanic: - c.Level.SetLevel(zap.PanicLevel) - default: - return errors.Errorf("invalid level: %s", level) + lvl, err := ParseLoggerLevel(level) + if err != nil { + return err } + c.Level.SetLevel(lvl) return nil } } @@ -219,30 +231,23 @@ func NewLogger(optfs ...LoggerOption) (l *LoggerType, err error) { } // Level get current level of logger -func (l *LoggerType) Level() LoggerLevel { - return LoggerLevel(l.level.String()) +func (l *LoggerType) Level() zapcore.Level { + return l.level.Level() } // ChangeLevel change logger level -func (l *LoggerType) ChangeLevel(level LoggerLevel) (err error) { - switch level { - case LoggerLevelDebug: - l.level.SetLevel(zap.DebugLevel) - case LoggerLevelInfo: - l.level.SetLevel(zap.InfoLevel) - case LoggerLevelWarn: - l.level.SetLevel(zap.WarnLevel) - case LoggerLevelError: - l.level.SetLevel(zap.ErrorLevel) - case LoggerLevelPanic: - l.level.SetLevel(zap.PanicLevel) - case LoggerLevelFatal: - l.level.SetLevel(zap.FatalLevel) - default: - return errors.Errorf("invalid level %s", string(level)) +// +// all children logger share the same level of their parent logger, +// so if you change any logger's level, all its parent and +// children logger's level will be changed. +func (l *LoggerType) ChangeLevel(level string) (err error) { + lvl, err := ParseLoggerLevel(level) + if err != nil { + return err } - l.Debug("set logger level", zap.String("level", level.String())) + l.level.SetLevel(lvl) + l.Debug("set logger level", zap.String("level", level)) return } diff --git a/logger_test.go b/logger_test.go index 895d1be..154e679 100644 --- a/logger_test.go +++ b/logger_test.go @@ -7,6 +7,7 @@ import ( "time" zap "github.com/Laisky/zap" + "github.com/Laisky/zap/zapcore" "github.com/stretchr/testify/require" // zap "github.com/Laisky/zap" ) @@ -16,7 +17,7 @@ func TestNewLogger(t *testing.T) { require.NoError(t, err) lvl := logger.Level() - require.Equal(t, LoggerLevelDebug, lvl) + require.Equal(t, zap.DebugLevel, lvl) _, err = NewLogger() require.NoError(t, err) @@ -218,3 +219,52 @@ func ExampleAlertPusher() { // time.Sleep(1 * time.Second) // t.Error() // } + +func TestChangeLoggerLevel(t *testing.T) { + var allLogs []string + logger, err := NewLogger( + WithLoggerZapOptions(zap.Hooks(func(e zapcore.Entry) error { + allLogs = append(allLogs, e.Message) + return nil + })), + WithLoggerLevel(LoggerLevelDebug), + ) + require.NoError(t, err) + + // case: normal log + { + msg := RandomStringWithLength(50) + logger.Debug(msg) + require.Equal(t, msg, allLogs[len(allLogs)-1]) + } + + // case: change level + { + msg := RandomStringWithLength(50) + err = logger.ChangeLevel(LoggerLevelInfo) + require.NoError(t, err) + logger.Debug(msg) + require.Len(t, allLogs, 1) + require.NotEqual(t, msg, allLogs[len(allLogs)-1]) + logger.ChangeLevel(LoggerLevelDebug) + } + + // case: change level for child logger + { + msg := RandomStringWithLength(50) + childLogger := logger.Clone() + err = childLogger.ChangeLevel(LoggerLevelInfo) + require.NoError(t, err) + logger.Debug(msg) + require.NotEqual(t, msg, allLogs[len(allLogs)-1]) + + msg = RandomStringWithLength(50) + childLogger.Info(msg) + require.Equal(t, msg, allLogs[len(allLogs)-1]) + + msg = RandomStringWithLength(50) + childLogger.Debug(msg) + require.NotEqual(t, msg, allLogs[len(allLogs)-1]) + } + +} diff --git a/utils.go b/utils.go index 5a6d16e..f1db0a1 100644 --- a/utils.go +++ b/utils.go @@ -527,9 +527,6 @@ func (c *SimpleExpCache) Get() (data interface{}, ok bool) { c.mu.RLock() data = c.data - fmt.Println("c.expiredAt", c.expiredAt) - fmt.Println("Clock.GetUTCNow()", Clock.GetUTCNow()) - ok = Clock.GetUTCNow().Before(c.expiredAt) c.mu.RUnlock() From 1463ddf3f76d00d7a71d349ba4db77a09b92eee7 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Fri, 27 Aug 2021 05:52:51 +0000 Subject: [PATCH 04/14] feat: add `FIFO` --- fifo.go | 84 +++++++++++++++++++++++++++++++++++++++ fifo_test.go | 99 ++++++++++++++++++++++++++++++++++++++++++++++ jwt.go | 13 +++--- structures/heap.go | 1 - 4 files changed, 191 insertions(+), 6 deletions(-) create mode 100644 fifo.go create mode 100644 fifo_test.go diff --git a/fifo.go b/fifo.go new file mode 100644 index 0000000..6c703d3 --- /dev/null +++ b/fifo.go @@ -0,0 +1,84 @@ +package utils + +import ( + "sync/atomic" + "unsafe" +) + +type fifoNode struct { + next *unsafe.Pointer + d interface{} +} + +// FIFO is a lock-free First-In-First-Out queue +type FIFO struct { + head *unsafe.Pointer + tail *unsafe.Pointer + len int64 +} + +// NewFIFO create a new FIFO queue +func NewFIFO() *FIFO { + var next unsafe.Pointer + node := &fifoNode{ + next: &next, + d: "laisky", + } + head := unsafe.Pointer(node) + tail := unsafe.Pointer(node) + return &FIFO{ + head: &head, + tail: &tail, + } +} + +// Put put an data into queue's tail +func (f *FIFO) Put(d interface{}) { + var next unsafe.Pointer + newNode := &fifoNode{ + d: d, + next: &next, + } + newAddr := unsafe.Pointer(newNode) + + var tailAddr unsafe.Pointer + for { + tailAddr = atomic.LoadPointer(f.tail) + tailNode := (*fifoNode)(tailAddr) + if atomic.CompareAndSwapPointer(tailNode.next, unsafe.Pointer(uintptr(0)), newAddr) { + atomic.AddInt64(&f.len, 1) + break + } + + atomic.CompareAndSwapPointer(f.tail, tailAddr, atomic.LoadPointer(tailNode.next)) + } + + atomic.CompareAndSwapPointer(f.tail, tailAddr, newAddr) +} + +// Get pop data from the head of queue +func (f *FIFO) Get() interface{} { + var nextNode *fifoNode + for { + headAddr := atomic.LoadPointer(f.head) + headNode := (*fifoNode)(headAddr) + nextAddr := atomic.LoadPointer(headNode.next) + if nextAddr == unsafe.Pointer(uintptr(0)) { + // queue is empty + return nil + } + + nextNode = (*fifoNode)(nextAddr) + if atomic.CompareAndSwapPointer(f.head, headAddr, nextAddr) { + atomic.AddInt64(&f.len, -1) + break + } + } + + return nextNode.d +} + +// Len return the length of queue +func (f *FIFO) Len() int { + return int(atomic.LoadInt64(&f.len)) +} diff --git a/fifo_test.go b/fifo_test.go new file mode 100644 index 0000000..978676c --- /dev/null +++ b/fifo_test.go @@ -0,0 +1,99 @@ +package utils + +import ( + "fmt" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + "unsafe" + + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +func ExampleFIFO() { + f := NewFIFO() + f.Put(1) + v := f.Get() + fmt.Println(v) + // Output: 1 +} + +func Test_UnsafePtr(t *testing.T) { + var a int + + addr := unsafe.Pointer(&a) + + b := *(*int)(atomic.LoadPointer(&addr)) + require.Equal(t, a, b) +} + +func TestNewFIFO(t *testing.T) { + f := NewFIFO() + var pool errgroup.Group + start := make(chan struct{}) + + var mu sync.Mutex + var cnt int32 + var got []interface{} + + for i := 0; i < 100; i++ { + pool.Go(func() error { + <-start + for i := 0; i < 100; i++ { + switch rand.Intn(2) { + case 0: + f.Put(i) + atomic.AddInt32(&cnt, 1) + case 1: + v := f.Get() + if v != nil { + mu.Lock() + got = append(got, v) + mu.Unlock() + } + } + } + + return nil + }) + } + + time.Sleep(time.Second) + close(start) + err := pool.Wait() + require.NoError(t, err) + f.Len() + + for { + v := f.Get() + if v == nil { + break + } + + got = append(got, v) + } + + require.Equal(t, 0, f.Len()) + require.Len(t, got, int(cnt)) +} + +// BenchmarkFIFO +// +// cpu: Intel(R) Core(TM) i7-4790 CPU @ 3.60GHz +// BenchmarkFIFO-8 368847 3330 ns/op 15 B/op 0 allocs/op +func BenchmarkFIFO(b *testing.B) { + f := NewFIFO() + b.RunParallel(func(p *testing.PB) { + for p.Next() { + switch rand.Intn(2) { + case 0: + f.Put(2) + case 1: + _ = f.Get() + } + } + }) +} diff --git a/jwt.go b/jwt.go index eaf6a40..9a583fb 100644 --- a/jwt.go +++ b/jwt.go @@ -18,15 +18,18 @@ var ( // ParseJWTTokenWithoutValidate parse and get payload without validate jwt token func ParseJWTTokenWithoutValidate(token string) (payload jwt.MapClaims, err error) { - var jt *jwt.Token - if jt, err = jwt.Parse(token, func(_ *jwt.Token) (interface{}, error) { + jt, err := jwt.Parse(token, func(_ *jwt.Token) (interface{}, error) { return "", nil - }); jt == nil && err != nil { + }) + if err != nil { return nil, errors.Wrap(err, "parse jwt token") } + if jt == nil { + return nil, errors.Errorf("token is nil") + } - var ok bool - if payload, ok = jt.Claims.(jwt.MapClaims); !ok { + payload, ok := jt.Claims.(jwt.MapClaims) + if !ok { return nil, errors.New("payload type not match `map[string]interface{}`") } diff --git a/structures/heap.go b/structures/heap.go index 47fd032..08f685d 100644 --- a/structures/heap.go +++ b/structures/heap.go @@ -74,7 +74,6 @@ func (p *PriorityQ) Push(x interface{}) { // Remove remove an specific item func (p *PriorityQ) Remove(v HeapItemItf) (ok bool) { - // utils.Logger.Debug("remove item") for i, it := range p.q { if it == v { p.q = append(p.q[:i], p.q[i+1:]...) From 45d3088a269d4fd1a66c1807de32c3fa97b57642 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Fri, 27 Aug 2021 06:34:39 +0000 Subject: [PATCH 05/14] docs: add some comment --- fifo.go | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/fifo.go b/fifo.go index 6c703d3..561621d 100644 --- a/fifo.go +++ b/fifo.go @@ -6,70 +6,68 @@ import ( ) type fifoNode struct { - next *unsafe.Pointer + next unsafe.Pointer d interface{} } // FIFO is a lock-free First-In-First-Out queue +// +// paper: https://1drv.ms/b/s!Au45o0W1gVVLuNxYkPzfBo4fOssFPQ?e=TYxHKl type FIFO struct { - head *unsafe.Pointer - tail *unsafe.Pointer + head unsafe.Pointer + tail unsafe.Pointer len int64 } // NewFIFO create a new FIFO queue func NewFIFO() *FIFO { - var next unsafe.Pointer - node := &fifoNode{ - next: &next, - d: "laisky", + // add a dummy node to the queue to avoid contention betweet head & tail + // when queue is empty + dummyNonde := &fifoNode{ + d: "laisky", } - head := unsafe.Pointer(node) - tail := unsafe.Pointer(node) return &FIFO{ - head: &head, - tail: &tail, + head: unsafe.Pointer(dummyNonde), + tail: unsafe.Pointer(dummyNonde), } } // Put put an data into queue's tail func (f *FIFO) Put(d interface{}) { - var next unsafe.Pointer newNode := &fifoNode{ - d: d, - next: &next, + d: d, } newAddr := unsafe.Pointer(newNode) var tailAddr unsafe.Pointer for { - tailAddr = atomic.LoadPointer(f.tail) + tailAddr = atomic.LoadPointer(&f.tail) tailNode := (*fifoNode)(tailAddr) - if atomic.CompareAndSwapPointer(tailNode.next, unsafe.Pointer(uintptr(0)), newAddr) { + if atomic.CompareAndSwapPointer(&tailNode.next, unsafe.Pointer(uintptr(0)), newAddr) { atomic.AddInt64(&f.len, 1) break } - atomic.CompareAndSwapPointer(f.tail, tailAddr, atomic.LoadPointer(tailNode.next)) + atomic.CompareAndSwapPointer(&f.tail, tailAddr, atomic.LoadPointer(&tailNode.next)) } - atomic.CompareAndSwapPointer(f.tail, tailAddr, newAddr) + atomic.CompareAndSwapPointer(&f.tail, tailAddr, newAddr) } // Get pop data from the head of queue func (f *FIFO) Get() interface{} { var nextNode *fifoNode for { - headAddr := atomic.LoadPointer(f.head) + headAddr := atomic.LoadPointer(&f.head) headNode := (*fifoNode)(headAddr) - nextAddr := atomic.LoadPointer(headNode.next) + nextAddr := atomic.LoadPointer(&headNode.next) if nextAddr == unsafe.Pointer(uintptr(0)) { // queue is empty return nil } nextNode = (*fifoNode)(nextAddr) - if atomic.CompareAndSwapPointer(f.head, headAddr, nextAddr) { + if atomic.CompareAndSwapPointer(&f.head, headAddr, nextAddr) { atomic.AddInt64(&f.len, -1) break } From e836dccade519aa6f4cbb5bab2933e4d1f401929 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Fri, 27 Aug 2021 08:59:19 +0000 Subject: [PATCH 06/14] docs: add some comment --- fifo.go | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/fifo.go b/fifo.go index 561621d..88da598 100644 --- a/fifo.go +++ b/fifo.go @@ -14,28 +14,42 @@ type fifoNode struct { // // paper: https://1drv.ms/b/s!Au45o0W1gVVLuNxYkPzfBo4fOssFPQ?e=TYxHKl type FIFO struct { + // head the node that before real head node + // + // head.next is the real head node + // + // unsafe.pointer will tell gc not to remove object in heap head unsafe.Pointer + // tail maybe(maynot) the tail node in queue tail unsafe.Pointer len int64 } +// add a dummy node to the queue to avoid contention +// betweet head & tail when queue is empty +// +// dummyNode is the default value to unsafe.pointer as an empty pointer +var dummyNode = &fifoNode{ + d: "dummy", +} + +func init() { + dummyNode.next = unsafe.Pointer(dummyNode) +} + // NewFIFO create a new FIFO queue func NewFIFO() *FIFO { - // add a dummy node to the queue to avoid contention betweet head & tail - // when queue is empty - dummyNonde := &fifoNode{ - d: "laisky", - } return &FIFO{ - head: unsafe.Pointer(dummyNonde), - tail: unsafe.Pointer(dummyNonde), + head: unsafe.Pointer(dummyNode), + tail: unsafe.Pointer(dummyNode), } } // Put put an data into queue's tail func (f *FIFO) Put(d interface{}) { newNode := &fifoNode{ - d: d, + d: d, + next: unsafe.Pointer(dummyNode), } newAddr := unsafe.Pointer(newNode) @@ -43,11 +57,12 @@ func (f *FIFO) Put(d interface{}) { for { tailAddr = atomic.LoadPointer(&f.tail) tailNode := (*fifoNode)(tailAddr) - if atomic.CompareAndSwapPointer(&tailNode.next, unsafe.Pointer(uintptr(0)), newAddr) { + if atomic.CompareAndSwapPointer(&tailNode.next, unsafe.Pointer(dummyNode), newAddr) { atomic.AddInt64(&f.len, 1) break } + // tail may not be the exact tail node, so we need to check again atomic.CompareAndSwapPointer(&f.tail, tailAddr, atomic.LoadPointer(&tailNode.next)) } @@ -61,7 +76,7 @@ func (f *FIFO) Get() interface{} { headAddr := atomic.LoadPointer(&f.head) headNode := (*fifoNode)(headAddr) nextAddr := atomic.LoadPointer(&headNode.next) - if nextAddr == unsafe.Pointer(uintptr(0)) { + if nextAddr == unsafe.Pointer(dummyNode) { // queue is empty return nil } From 4551ad38f162e2d0cf7dc73b532d1de1a3a8ddbe Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Fri, 27 Aug 2021 09:41:04 +0000 Subject: [PATCH 07/14] fix: fix jwt & fifo bug --- fifo.go | 26 +++++++++++++------------- fifo_test.go | 10 +++++++--- jwt.go | 20 +++----------------- jwt_test.go | 29 ++++++++++++----------------- 4 files changed, 35 insertions(+), 50 deletions(-) diff --git a/fifo.go b/fifo.go index 88da598..97479d2 100644 --- a/fifo.go +++ b/fifo.go @@ -25,20 +25,20 @@ type FIFO struct { len int64 } -// add a dummy node to the queue to avoid contention -// betweet head & tail when queue is empty -// -// dummyNode is the default value to unsafe.pointer as an empty pointer -var dummyNode = &fifoNode{ - d: "dummy", -} - -func init() { - dummyNode.next = unsafe.Pointer(dummyNode) +// emptyNode is the default value to unsafe.pointer as an empty pointer +var emptyNode = &fifoNode{ + d: "empty", } // NewFIFO create a new FIFO queue func NewFIFO() *FIFO { + // add a dummy node to the queue to avoid contention + // betweet head & tail when queue is empty + var dummyNode = &fifoNode{ + d: "dummy", + next: unsafe.Pointer(emptyNode), + } + return &FIFO{ head: unsafe.Pointer(dummyNode), tail: unsafe.Pointer(dummyNode), @@ -49,7 +49,7 @@ func NewFIFO() *FIFO { func (f *FIFO) Put(d interface{}) { newNode := &fifoNode{ d: d, - next: unsafe.Pointer(dummyNode), + next: unsafe.Pointer(emptyNode), } newAddr := unsafe.Pointer(newNode) @@ -57,7 +57,7 @@ func (f *FIFO) Put(d interface{}) { for { tailAddr = atomic.LoadPointer(&f.tail) tailNode := (*fifoNode)(tailAddr) - if atomic.CompareAndSwapPointer(&tailNode.next, unsafe.Pointer(dummyNode), newAddr) { + if atomic.CompareAndSwapPointer(&tailNode.next, unsafe.Pointer(emptyNode), newAddr) { atomic.AddInt64(&f.len, 1) break } @@ -76,7 +76,7 @@ func (f *FIFO) Get() interface{} { headAddr := atomic.LoadPointer(&f.head) headNode := (*fifoNode)(headAddr) nextAddr := atomic.LoadPointer(&headNode.next) - if nextAddr == unsafe.Pointer(dummyNode) { + if nextAddr == unsafe.Pointer(emptyNode) { // queue is empty return nil } diff --git a/fifo_test.go b/fifo_test.go index 978676c..70016ac 100644 --- a/fifo_test.go +++ b/fifo_test.go @@ -17,7 +17,11 @@ func ExampleFIFO() { f := NewFIFO() f.Put(1) v := f.Get() - fmt.Println(v) + if v == nil { + panic(v) + } + + fmt.Println(v.(int)) // Output: 1 } @@ -76,8 +80,8 @@ func TestNewFIFO(t *testing.T) { got = append(got, v) } - require.Equal(t, 0, f.Len()) - require.Len(t, got, int(cnt)) + require.Equal(t, 0, f.Len(), "empty") + require.Len(t, got, int(cnt), "total len") } // BenchmarkFIFO diff --git a/jwt.go b/jwt.go index 9a583fb..e5af9ca 100644 --- a/jwt.go +++ b/jwt.go @@ -17,23 +17,9 @@ var ( ) // ParseJWTTokenWithoutValidate parse and get payload without validate jwt token -func ParseJWTTokenWithoutValidate(token string) (payload jwt.MapClaims, err error) { - jt, err := jwt.Parse(token, func(_ *jwt.Token) (interface{}, error) { - return "", nil - }) - if err != nil { - return nil, errors.Wrap(err, "parse jwt token") - } - if jt == nil { - return nil, errors.Errorf("token is nil") - } - - payload, ok := jt.Claims.(jwt.MapClaims) - if !ok { - return nil, errors.New("payload type not match `map[string]interface{}`") - } - - return payload, nil +func ParseJWTTokenWithoutValidate(token string, payload jwt.Claims) (err error) { + _, _, err = new(jwt.Parser).ParseUnverified(token, payload) + return err } // JWT is token utils that support HS256/ES256 diff --git a/jwt_test.go b/jwt_test.go index 74d1430..bd31bf8 100644 --- a/jwt_test.go +++ b/jwt_test.go @@ -67,17 +67,13 @@ func TestJWTSignAndVerify(t *testing.T) { WithJWTPubKeyByte(es256PubByte), WithJWTPriKeyByte(es256PriByte), ) - if err != nil { - t.Fatalf("got error: %+v", err) - } + require.NoError(t, err) jwtHS256, err := NewJWT( WithJWTSignMethod(SignMethodHS256), WithJWTSecretByte(secret), ) - if err != nil { - t.Fatalf("got error: %+v", err) - } + require.NoError(t, err) for _, j := range []*JWT{ jwtES256, @@ -93,9 +89,8 @@ func TestJWTSignAndVerify(t *testing.T) { // test sign & parse token, err := j.Sign(claims) - if err != nil { - t.Fatalf("generate token error %+v", err) - } + require.NoError(t, err) + // expect := "eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJkdW5lIiwic3ViIjoibGFpc2t5In0.UtcJn1th7rvZNr0HLl6h5G8XE-sJLVSqyc96LYAFG42-p0ZAJJeDeE_9a5sp770hEaIXMtZSvVeeBQre90oTLA" // if token != expect { // t.Fatalf("expect %v,\n got %v", expect, token) @@ -152,14 +147,13 @@ func TestJWTSignAndVerify(t *testing.T) { } func TestParseJWTTokenWithoutValidate(t *testing.T) { - token := "eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJkdW5lIiwic3ViIjoibGFpc2t5In0.UtcJn1th7rvZNr0HLl6h5G8XE-sJLVSqyc96LYAFG42-p0ZAJJeDeE_9a5sp770hEaIXMtZSvVeeBQre90oTLA" - claims, err := ParseJWTTokenWithoutValidate(token) - require.NoError(t, err) + token := "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOlsiZHVuZSJdLCJzdWIiOiJsYWlza3kifQ.cYnd2OdN-i3kuPXSUc4xj1rkVk5elJnxln6zDdvlOUc" - if claims["sub"] != "laisky" || - claims["aud"] != "dune" { - t.Fatal() - } + c := new(jwt.StandardClaims) + err := ParseJWTTokenWithoutValidate(token, c) + require.NoError(t, err) + require.Equal(t, "laisky", c.Subject) + require.Equal(t, []string{"dune"}, c.Audience) } // https://snyk.io/vuln/SNYK-GOLANG-GITHUBCOMDGRIJALVAJWTGO-596515?utm_medium=Partner&utm_source=RedHat&utm_campaign=Code-Ready-Analytics-2020&utm_content=vuln/SNYK-GOLANG-GITHUBCOMDGRIJALVAJWTGO-596515 @@ -190,7 +184,8 @@ func TestJWTAudValunerable(t *testing.T) { // bug: slice aud will bypass verify { - claims, err := ParseJWTTokenWithoutValidate(token) + claims := new(jwt.StandardClaims) + err := ParseJWTTokenWithoutValidate(token, claims) require.NoError(t, err) ok := claims.VerifyAudience("laisky", false) From a26b447dacddbd4b5448d1d94688f84543762459 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Mon, 30 Aug 2021 02:37:08 +0000 Subject: [PATCH 08/14] fix: aba problem --- fifo.go | 53 ++++++++++++++++++++++++++++++++++++++++------------ fifo_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 84 insertions(+), 20 deletions(-) diff --git a/fifo.go b/fifo.go index 97479d2..cde6e32 100644 --- a/fifo.go +++ b/fifo.go @@ -1,13 +1,34 @@ package utils import ( + "sync" "sync/atomic" "unsafe" ) +var fifoPool = sync.Pool{ + New: func() interface{} { + return &fifoNode{ + next: unsafe.Pointer(emptyNode), + } + }, +} + type fifoNode struct { next unsafe.Pointer d interface{} + // refcnt to avoid ABA problem + refcnt int32 +} + +// AddRef add ref count +func (f *fifoNode) AddRef(n int32) { + atomic.AddInt32(&f.refcnt, n) +} + +// Refcnt get ref count +func (f *fifoNode) Refcnt() int32 { + return atomic.LoadInt32(&f.refcnt) } // FIFO is a lock-free First-In-First-Out queue @@ -34,10 +55,9 @@ var emptyNode = &fifoNode{ func NewFIFO() *FIFO { // add a dummy node to the queue to avoid contention // betweet head & tail when queue is empty - var dummyNode = &fifoNode{ - d: "dummy", - next: unsafe.Pointer(emptyNode), - } + var dummyNode = fifoPool.Get().(*fifoNode) + dummyNode.d = "dummy" + dummyNode.next = unsafe.Pointer(emptyNode) return &FIFO{ head: unsafe.Pointer(dummyNode), @@ -47,10 +67,16 @@ func NewFIFO() *FIFO { // Put put an data into queue's tail func (f *FIFO) Put(d interface{}) { - newNode := &fifoNode{ - d: d, - next: unsafe.Pointer(emptyNode), + var newNode *fifoNode + for { + newNode = fifoPool.Get().(*fifoNode) + if newNode.Refcnt() == 0 { + break + } } + + newNode.d = d + newNode.next = unsafe.Pointer(emptyNode) newAddr := unsafe.Pointer(newNode) var tailAddr unsafe.Pointer @@ -71,7 +97,6 @@ func (f *FIFO) Put(d interface{}) { // Get pop data from the head of queue func (f *FIFO) Get() interface{} { - var nextNode *fifoNode for { headAddr := atomic.LoadPointer(&f.head) headNode := (*fifoNode)(headAddr) @@ -81,14 +106,18 @@ func (f *FIFO) Get() interface{} { return nil } - nextNode = (*fifoNode)(nextAddr) + headNode.AddRef(1) + nextNode := (*fifoNode)(nextAddr) if atomic.CompareAndSwapPointer(&f.head, headAddr, nextAddr) { + // do not release refcnt atomic.AddInt64(&f.len, -1) - break + return nextNode.d } - } - return nextNode.d + // release refcnt when skip node + headNode.AddRef(-1) + fifoPool.Put(headNode) + } } // Len return the length of queue diff --git a/fifo_test.go b/fifo_test.go index 70016ac..1fc8d89 100644 --- a/fifo_test.go +++ b/fifo_test.go @@ -89,15 +89,50 @@ func TestNewFIFO(t *testing.T) { // cpu: Intel(R) Core(TM) i7-4790 CPU @ 3.60GHz // BenchmarkFIFO-8 368847 3330 ns/op 15 B/op 0 allocs/op func BenchmarkFIFO(b *testing.B) { - f := NewFIFO() - b.RunParallel(func(p *testing.PB) { - for p.Next() { - switch rand.Intn(2) { - case 0: + b.Run("fifo", func(b *testing.B) { + f := NewFIFO() + b.RunParallel(func(p *testing.PB) { + for p.Next() { + switch rand.Intn(2) { + case 0: + f.Put(2) + case 1: + _ = f.Get() + } + } + }) + }) +} + +func BenchmarkFIFOAndChan(b *testing.B) { + + b.Run("fifo", func(b *testing.B) { + f := NewFIFO() + b.RunParallel(func(p *testing.PB) { + for p.Next() { f.Put(2) - case 1: - _ = f.Get() + f.Get() } - } + }) + }) + + b.Run("channel struct", func(b *testing.B) { + ch := make(chan struct{}, 10) + b.RunParallel(func(p *testing.PB) { + for p.Next() { + ch <- struct{}{} + <-ch + } + }) + }) + + b.Run("channel int", func(b *testing.B) { + ch := make(chan int, 10) + b.RunParallel(func(p *testing.PB) { + for p.Next() { + ch <- 2 + <-ch + } + }) }) } From 521d33369b9f7d1a0c8787e3f76e12204fbf7988 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Mon, 30 Aug 2021 03:34:44 +0000 Subject: [PATCH 09/14] perf: improve fifo pool --- .gitignore | 1 + fifo.go | 24 ++++++++++++++++++++---- fifo_test.go | 40 ++++++++++++++++++++++++++++++++++------ 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index d51f833..b18d522 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,7 @@ tags memory.pprof cpu.pprof coverage.txt +profile.out # fuzz corpus diff --git a/fifo.go b/fifo.go index cde6e32..bcb74a0 100644 --- a/fifo.go +++ b/fifo.go @@ -97,8 +97,28 @@ func (f *FIFO) Put(d interface{}) { // Get pop data from the head of queue func (f *FIFO) Get() interface{} { + var oldHeadAddr unsafe.Pointer for { headAddr := atomic.LoadPointer(&f.head) + for { + if oldHeadAddr == nil { + oldHeadAddr = headAddr + break + } + + if oldHeadAddr == headAddr { + break + } + + oldHeadNode := (*fifoNode)(oldHeadAddr) + if !atomic.CompareAndSwapInt32(&oldHeadNode.refcnt, 1, 0) { + break + } + + oldHeadAddr = atomic.LoadPointer(&oldHeadNode.next) + fifoPool.Put(oldHeadNode) + } + headNode := (*fifoNode)(headAddr) nextAddr := atomic.LoadPointer(&headNode.next) if nextAddr == unsafe.Pointer(emptyNode) { @@ -113,10 +133,6 @@ func (f *FIFO) Get() interface{} { atomic.AddInt64(&f.len, -1) return nextNode.d } - - // release refcnt when skip node - headNode.AddRef(-1) - fifoPool.Put(headNode) } } diff --git a/fifo_test.go b/fifo_test.go index 1fc8d89..30870a6 100644 --- a/fifo_test.go +++ b/fifo_test.go @@ -110,8 +110,14 @@ func BenchmarkFIFOAndChan(b *testing.B) { f := NewFIFO() b.RunParallel(func(p *testing.PB) { for p.Next() { - f.Put(2) - f.Get() + for p.Next() { + switch rand.Intn(2) { + case 0: + f.Put(2) + case 1: + _ = f.Get() + } + } } }) }) @@ -120,8 +126,20 @@ func BenchmarkFIFOAndChan(b *testing.B) { ch := make(chan struct{}, 10) b.RunParallel(func(p *testing.PB) { for p.Next() { - ch <- struct{}{} - <-ch + for p.Next() { + switch rand.Intn(2) { + case 0: + select { + case ch <- struct{}{}: + default: + } + case 1: + select { + case <-ch: + default: + } + } + } } }) }) @@ -130,8 +148,18 @@ func BenchmarkFIFOAndChan(b *testing.B) { ch := make(chan int, 10) b.RunParallel(func(p *testing.PB) { for p.Next() { - ch <- 2 - <-ch + switch rand.Intn(2) { + case 0: + select { + case ch <- 2: + default: + } + case 1: + select { + case <-ch: + default: + } + } } }) }) From d77c00785cc85c94a0f8ec6d993819284d063c8b Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Mon, 30 Aug 2021 06:45:19 +0000 Subject: [PATCH 10/14] perf: improve fifo pool --- fifo.go | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/fifo.go b/fifo.go index bcb74a0..aa5a632 100644 --- a/fifo.go +++ b/fifo.go @@ -1,6 +1,7 @@ package utils import ( + "math" "sync" "sync/atomic" "unsafe" @@ -22,8 +23,8 @@ type fifoNode struct { } // AddRef add ref count -func (f *fifoNode) AddRef(n int32) { - atomic.AddInt32(&f.refcnt, n) +func (f *fifoNode) AddRef(n int32) int32 { + return atomic.AddInt32(&f.refcnt, n) } // Refcnt get ref count @@ -97,40 +98,26 @@ func (f *FIFO) Put(d interface{}) { // Get pop data from the head of queue func (f *FIFO) Get() interface{} { - var oldHeadAddr unsafe.Pointer for { headAddr := atomic.LoadPointer(&f.head) - for { - if oldHeadAddr == nil { - oldHeadAddr = headAddr - break - } - - if oldHeadAddr == headAddr { - break - } - - oldHeadNode := (*fifoNode)(oldHeadAddr) - if !atomic.CompareAndSwapInt32(&oldHeadNode.refcnt, 1, 0) { - break - } - - oldHeadAddr = atomic.LoadPointer(&oldHeadNode.next) - fifoPool.Put(oldHeadNode) + headNode := (*fifoNode)(headAddr) + if headNode.AddRef(1) < 0 { + headNode.AddRef(-1) + continue } - headNode := (*fifoNode)(headAddr) nextAddr := atomic.LoadPointer(&headNode.next) if nextAddr == unsafe.Pointer(emptyNode) { // queue is empty return nil } - headNode.AddRef(1) nextNode := (*fifoNode)(nextAddr) if atomic.CompareAndSwapPointer(&f.head, headAddr, nextAddr) { // do not release refcnt atomic.AddInt64(&f.len, -1) + atomic.StoreInt32(&headNode.refcnt, math.MinInt32) + fifoPool.Put(headNode) return nextNode.d } } From 0d2538c01c9a738fe99188ddc14d21bfbbf8e2b6 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Fri, 3 Sep 2021 09:36:22 +0000 Subject: [PATCH 11/14] feat: add skiplist --- go.mod | 3 +++ go.sum | 2 ++ structures/heap.go => heap.go | 19 +++++++-------- structures/heap_test.go => heap_test.go | 2 +- consistenthash/jumphash.go => jumphash.go | 3 +-- random.go | 5 ++++ skiplist.go | 7 ++++++ skiplist_test.go | 29 +++++++++++++++++++++++ 8 files changed, 57 insertions(+), 13 deletions(-) rename structures/heap.go => heap.go (90%) rename structures/heap_test.go => heap_test.go (99%) rename consistenthash/jumphash.go => jumphash.go (83%) create mode 100644 skiplist.go create mode 100644 skiplist_test.go diff --git a/go.mod b/go.mod index eb91e47..5ff882b 100644 --- a/go.mod +++ b/go.mod @@ -22,4 +22,7 @@ require ( golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df + skiplist v0.0.0-00010101000000-000000000000 ) + +replace skiplist => github.com/Laisky/fast-skiplist v0.0.0-20210903093518-aea9c86c0252 diff --git a/go.sum b/go.sum index 0412211..e44ac45 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Laisky/fast-skiplist v0.0.0-20210903093518-aea9c86c0252 h1:5MGOoFlNUR7PX3ppBzzg5nSxy4eDFpWaZZYAfb9ktKk= +github.com/Laisky/fast-skiplist v0.0.0-20210903093518-aea9c86c0252/go.mod h1:1qPOW+o/q+5G/cjALu4aG3CKmwVomSwqlL5FnoJanOE= github.com/Laisky/go-chaining v0.0.0-20180507092046-43dcdc5a21be h1:7Rxhm6IjOtDAyj8eScOFntevwzkWhx94zi48lxo4m4w= github.com/Laisky/go-chaining v0.0.0-20180507092046-43dcdc5a21be/go.mod h1:1mdzaETo0kjvCQICPSePsoaatJN4l7JvEA1200lyevo= github.com/Laisky/graphql v1.0.5 h1:8eJ7mrXKVkKxZ+Nw1HPs3iQPVNxXGctysqTEY0lNBlc= diff --git a/structures/heap.go b/heap.go similarity index 90% rename from structures/heap.go rename to heap.go index 08f685d..4cd91b8 100644 --- a/structures/heap.go +++ b/heap.go @@ -1,10 +1,9 @@ -package structures +package utils import ( "container/heap" "fmt" - utils "github.com/Laisky/go-utils" "github.com/Laisky/zap" ) @@ -36,7 +35,7 @@ type PriorityQ struct { // NewPriorityQ create new PriorityQ func NewPriorityQ(isMaxTop bool) *PriorityQ { - // utils.Logger.Debug("create PriorityQ") + // Logger.Debug("create PriorityQ") return &PriorityQ{ isMaxTop: isMaxTop, q: []HeapItemItf{}, @@ -45,13 +44,13 @@ func NewPriorityQ(isMaxTop bool) *PriorityQ { // Len get length of items in heapq func (p *PriorityQ) Len() int { - // utils.Logger.Debug("len", zap.Int("len", len(p.q))) + // Logger.Debug("len", zap.Int("len", len(p.q))) return len(p.q) } // Less compare two items in heapq func (p *PriorityQ) Less(i, j int) bool { - // utils.Logger.Debug("less two items", zap.Int("i", i), zap.Int("j", j)) + // Logger.Debug("less two items", zap.Int("i", i), zap.Int("j", j)) if p.isMaxTop { return p.q[i].GetPriority() > p.q[j].GetPriority() } @@ -61,13 +60,13 @@ func (p *PriorityQ) Less(i, j int) bool { // Swap swat two items in heapq func (p *PriorityQ) Swap(i, j int) { - // utils.Logger.Debug("swap two items", zap.Int("i", i), zap.Int("j", j)) + // Logger.Debug("swap two items", zap.Int("i", i), zap.Int("j", j)) p.q[i], p.q[j] = p.q[j], p.q[i] } // Push push new item into heapq func (p *PriorityQ) Push(x interface{}) { - // utils.Logger.Debug("push item", zap.Int("priority", x.(HeapItemItf).GetPriority())) + // Logger.Debug("push item", zap.Int("priority", x.(HeapItemItf).GetPriority())) item := x.(HeapItemItf) p.q = append(p.q, item) } @@ -87,7 +86,7 @@ func (p *PriorityQ) Remove(v HeapItemItf) (ok bool) { // Pop pop from the tail. // if `isMaxTop=True`, pop the biggest item func (p *PriorityQ) Pop() (popped interface{}) { - utils.Logger.Debug("pop item") + Logger.Debug("pop item") n := len(p.q) popped = p.q[n-1] p.q[n-1] = nil // avoid memory leak @@ -116,7 +115,7 @@ func GetSmallestNItems(inputChan <-chan HeapItemItf, topN int) ([]HeapItemItf, e // * use min-heap to calculates topN Highest items. // * use max-heap to calculates topN Lowest items. func GetTopKItems(inputChan <-chan HeapItemItf, topN int, isHighest bool) ([]HeapItemItf, error) { - utils.Logger.Debug("GetMostFreqWords for key2PriMap", zap.Int("topN", topN)) + Logger.Debug("GetMostFreqWords for key2PriMap", zap.Int("topN", topN)) if topN < 2 { return nil, fmt.Errorf("GetMostFreqWords topN must larger than 2") } @@ -180,7 +179,7 @@ LOAD_LOOP: } } - utils.Logger.Debug("process all items", zap.Int("total", nTotal)) + Logger.Debug("process all items", zap.Int("total", nTotal)) for i := 1; i <= topN; i++ { // pop all needed items item = heap.Pop(p).(*itemType) items[topN-i] = item diff --git a/structures/heap_test.go b/heap_test.go similarity index 99% rename from structures/heap_test.go rename to heap_test.go index a83a4ec..5ee0763 100644 --- a/structures/heap_test.go +++ b/heap_test.go @@ -1,4 +1,4 @@ -package structures +package utils import ( "container/heap" diff --git a/consistenthash/jumphash.go b/jumphash.go similarity index 83% rename from consistenthash/jumphash.go rename to jumphash.go index 537113d..9937f0b 100644 --- a/consistenthash/jumphash.go +++ b/jumphash.go @@ -1,5 +1,4 @@ -// Package consistenthash contains some implementation of consistent hashing. -package consistenthash +package utils import "fmt" diff --git a/random.go b/random.go index 2bd64d7..00399c6 100644 --- a/random.go +++ b/random.go @@ -13,6 +13,11 @@ func init() { var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") +// NewRand new individual random to aviod global mutex +func NewRand() *rand.Rand { + return rand.New(rand.NewSource(time.Now().UnixNano())) +} + // RandomStringWithLength generate random string with specific length func RandomStringWithLength(n int) string { b := make([]rune, n) diff --git a/skiplist.go b/skiplist.go new file mode 100644 index 0000000..c76ac00 --- /dev/null +++ b/skiplist.go @@ -0,0 +1,7 @@ +package utils + +import "skiplist" + +func NewSkiplist() *skiplist.SkipList { + return skiplist.New() +} diff --git a/skiplist_test.go b/skiplist_test.go new file mode 100644 index 0000000..4eeaabe --- /dev/null +++ b/skiplist_test.go @@ -0,0 +1,29 @@ +package utils + +import ( + "math/rand" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewSkiplist(t *testing.T) { + l := NewSkiplist() + + var keys []float64 + for i := 0; i < 1000; i++ { + k := rand.Float64() + if v := l.Get(k); v != nil { + // do not overwrite + continue + } + + l.Set(k, k) + keys = append(keys, k) + } + + for i, k := range keys { + require.Equal(t, k, l.Get(k).Value().(float64), strconv.Itoa(i)) + } +} From 53660143deaded145efd45ebbbf8d77af37fe3a6 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Fri, 3 Sep 2021 09:57:40 +0000 Subject: [PATCH 12/14] style: fix lint warnning --- .golangci.lint.yml | 500 +++++++++++++++++++++++++++++++++++++++++++++ Makefile | 2 +- cmd/root.go | 1 + cmd/tls.go | 4 +- compressor.go | 17 +- counter.go | 3 - encrypt.go | 17 -- http.go | 16 +- logger.go | 17 +- logger_test.go | 3 +- net.go | 2 +- settings.go | 41 +--- settings_test.go | 14 +- sync.go | 35 ++-- throttle.go | 7 - throttle_test.go | 2 +- time.go | 16 +- time_test.go | 18 +- utils.go | 4 +- 19 files changed, 584 insertions(+), 135 deletions(-) create mode 100644 .golangci.lint.yml diff --git a/.golangci.lint.yml b/.golangci.lint.yml new file mode 100644 index 0000000..7e1b068 --- /dev/null +++ b/.golangci.lint.yml @@ -0,0 +1,500 @@ +# This file contains all available configuration options +# with their default values. +# 配置文件教程地址 https://golangci-lint.run/usage/configuration + +# options for analysis running +run: + # default concurrency is a available CPU number + concurrency: 4 + + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 5m + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: false + + # list of build tags, all linters use it. Default is empty list. + build-tags: + - mytag + + # which dirs to skip: issues from them won't be reported; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but default dirs are skipped independently + # from this option's value (see skip-dirs-use-default). + # "/" will be replaced by current OS file path separator to properly work + # on Windows. + skip-dirs: + - src/external_libs + - autogenerated_by_my_lib + - config + - pkg + - "^_.*" + + # default is true. Enables skipping of directories: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs-use-default: true + + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + # "/" will be replaced by current OS file path separator to properly work + # on Windows. + skip-files: + - ".*\\.log$" + - .gitlab-ci.yml + - .golangci.lint.yml + + # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules": + # If invoked with -mod=readonly, the go command is disallowed from the implicit + # automatic updating of go.mod described above. Instead, it fails when any changes + # to go.mod are needed. This setting is most useful to check that go.mod does + # not need updates, such as in a continuous integration and testing system. + # If invoked with -mod=vendor, the go command assumes that the vendor + # directory holds the correct copies of dependencies and ignores + # the dependency descriptions in go.mod. + # modules-download-mode: readonly|release|vendor + + # Allow multiple parallel golangci-lint instances running. + # If false (default) - golangci-lint acquires file lock on start. + allow-parallel-runners: false + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" + format: colored-line-number + + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + + # make issues output unique by line, default is true + uniq-by-line: true + + # add a prefix to the output file references; default is no prefix + path-prefix: "" + +# all available settings of specific linters +linters-settings: + dogsled: + # checks assignments with too many blank identifiers; default is 2 + max-blank-identifiers: 2 + dupl: + # tokens count to trigger issue, 150 by default + threshold: 100 + errcheck: + # report about not checking of errors in type assertions: `a := b.(MyStruct)`; + # default is false: such cases aren't reported by default. + check-type-assertions: false + + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: false + + # [deprecated] comma-separated list of pairs of the form pkg:regex + # the regex is used to ignore names within pkg. (default "fmt:.*"). + # see https://github.com/kisielk/errcheck#the-deprecated-method for details + ignore: fmt:.*,io/ioutil:^Read.* + + # path to a file containing a list of functions to exclude from checking + # see https://github.com/kisielk/errcheck#excluding-functions for details + exhaustive: + # indicates that switch statements are to be considered exhaustive if a + # 'default' case is present, even if all enum members aren't listed in the + # switch + default-signifies-exhaustive: false + funlen: + lines: 60 + statements: 40 + gci: + # put imports beginning with prefix after 3rd-party packages; + # only support one prefix + # if not set, use goimports.local-prefixes + local-prefixes: github.com/org/project + gocognit: + # minimal code complexity to report, 30 by default (but we recommend 10-20) + min-complexity: 30 + nestif: + # minimal complexity of if statements to report, 5 by default + min-complexity: 4 + goconst: + # minimal length of string constant, 3 by default + min-len: 3 + # minimal occurrences count to trigger, 3 by default + min-occurrences: 3 + gocritic: + # Which checks should be enabled; can't be combined with 'disabled-checks'; + # See https://go-critic.github.io/overview#checks-overview + # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` + # By default list of stable checks is used. + enabled-checks: + #- rangeValCopy + + # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty + disabled-checks: + - regexpMust + + # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks. + # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". + enabled-tags: + - performance + disabled-tags: + - experimental + + settings: # settings passed to gocritic + captLocal: # must be valid enabled check name + paramsOnly: true + rangeValCopy: + sizeThreshold: 32 + gocyclo: + # minimal code complexity to report, 30 by default (but we recommend 10-20) + min-complexity: 10 + godot: + # check all top-level comments, not only declarations + check-all: false + godox: + # report any comments starting with keywords, this is useful for TODO or FIXME comments that + # might be left in the code accidentally and should be resolved before merging + keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting + - NOTE + - OPTIMIZE # marks code that should be optimized before merging + - HACK # marks hack-arounds that should be removed before merging + gofmt: + # simplify code: gofmt with `-s` option, true by default + simplify: true + goheader: + values: + const: + # define here const type values in format k:v, for example: + # YEAR: 2020 + # COMPANY: MY COMPANY + regexp: + # define here regexp type values, for example + # AUTHOR: .*@mycompany\.com + template: + # put here copyright header template for source code files, for example: + # {{ AUTHOR }} {{ COMPANY }} {{ YEAR }} + # SPDX-License-Identifier: Apache-2.0 + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at: + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + template-path: + # also as alternative of directive 'template' you may put the path to file with the template source + goimports: + # put imports beginning with prefix after 3rd-party packages; + # it's a comma-separated list of prefixes + local-prefixes: github.com/org/project + golint: + # minimal confidence for issues, default is 0.8 + min-confidence: 0.8 + gomnd: + settings: + mnd: + # the list of enabled checks, see https://github.com/tommy-muehle/go-mnd/#checks for description. + checks: argument,case,condition,operation,return,assign + gomodguard: + allowed: + modules: # List of allowed modules + # - gopkg.in/yaml.v2 + domains: # List of allowed module domains + # - golang.org + blocked: + modules: # List of blocked modules + # - github.com/uudashr/go-module: # Blocked module + # recommendations: # Recommended modules that should be used instead (Optional) + # - golang.org/x/mod + # reason: "`mod` is the official go.mod parser library." # Reason why the recommended module should be used (Optional) + versions: # List of blocked module version constraints + # - github.com/mitchellh/go-homedir: # Blocked module with version constraint + # version: "< 1.1.0" # Version constraint, see https://github.com/Masterminds/semver#basic-comparisons + # reason: "testing if blocked version constraint works." # Reason why the version constraint exists. (Optional) + govet: + # report about shadowed variables + check-shadowing: true + + # settings per analyzer + settings: + printf: # analyzer name, run `go tool vet help` to see all analyzers + funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + + # enable or disable analyzers by name + enable: + - atomicalign + enable-all: false + disable: + - shadow + disable-all: false + depguard: + list-type: blacklist + include-go-root: false + packages: + # TODO: + # - github.com/sirupsen/logrus + packages-with-error-message: + # specify an error message to output when a blacklisted package is used + # - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" + lll: + # max line length, lines longer will be reported. Default is 120. + # '\t' is counted as 1 character by default, and can be changed with the tab-width option + line-length: 120 + # tab width in spaces. Default to 1. + tab-width: 4 + gosec: + excludes: + - G304 + maligned: + # print struct with more effective memory layout or not, false by default + suggest-new: true + misspell: + # Correct spellings using locale preferences for US or UK. + # Default is to use a neutral variety of English. + # Setting locale to US will correct the British spelling of 'colour' to 'color'. + locale: US + ignore-words: + - gorm + + nakedret: + # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 + max-func-lines: 30 + prealloc: + # XXX: we don't recommend using this linter before doing performance profiling. + # For most programs usage of prealloc will be a premature optimization. + + # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. + # True by default. + simple: true + range-loops: true # Report preallocation suggestions on range loops, true by default + for-loops: false # Report preallocation suggestions on for loops, false by default + nolintlint: + # Enable to ensure that nolint directives are all used. Default is true. + allow-unused: false + # Disable to ensure that nolint directives don't have a leading space. Default is true. + allow-leading-space: true + # Exclude following linters from requiring an explanation. Default is []. + allow-no-explanation: [] + # Enable to require an explanation of nonzero length after each nolint directive. Default is false. + require-explanation: true + # Enable to require nolint directives to mention the specific linter being suppressed. Default is false. + require-specific: true + rowserrcheck: + packages: + - github.com/jmoiron/sqlx + testpackage: + # regexp pattern to skip files + skip-regexp: (export|internal)_test\.go + unparam: + # Inspect exported functions, default is false. Set to true if no external program/library imports your code. + # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find external interfaces. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + unused: + # treat code as a program (not a library) and report unused exported identifiers; default is false. + # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find funcs usages. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + whitespace: + multi-if: false # Enforces newlines (or comments) after every multi-line if statement + multi-func: false # Enforces newlines (or comments) after every multi-line function signature + wsl: + # If true append is only allowed to be cuddled if appending value is + # matching variables, fields or types on line above. Default is true. + strict-append: true + # Allow calls and assignments to be cuddled as long as the lines have any + # matching variables, fields or types. Default is true. + allow-assign-and-call: true + # Allow multiline assignments to be cuddled. Default is true. + allow-multiline-assign: true + # Allow declarations (var) to be cuddled. + allow-cuddle-declarations: false + # Allow trailing comments in ending of blocks + allow-trailing-comment: false + # Force newlines in end of case at this limit (0 = never). + force-case-trailing-whitespace: 0 + # Force cuddling of err checks with err var assignment + force-err-cuddling: false + # Allow leading comments to be separated with empty liens + allow-separated-leading-comment: false + gofumpt: + # Choose whether or not to use the extra rules that are disabled + # by default + extra-rules: false + +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + # - dogsled + - dupl + - errcheck + # - exhaustive + # - funlen + # - gochecknoinits + - gocognit + - goconst + # - gocritic + # - gocyclo + - gofmt + # - goimports + - golint + # - gomnd + # - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + # - interfacer + - lll + - misspell + # - nakedret + # - noctx + # - nolintlint + # - rowserrcheck + # - scopelint + - staticcheck + - structcheck + - stylecheck + - typecheck + # - unconvert + # - unparam + - unused + # - varcheck + # - whitespace + fast: false + +issues: + # List of regexps of issue texts to exclude, empty list by default. + # But independently from this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. To list all + # excluded by default patterns execute `golangci-lint run --help` + exclude: + - "ineffectual assignment to `.*` (ineffassign)" #正则排除这lint 错误类型 + - 'shadow: declaration of ".*" shadows declaration' + - "^exported [^ ]* .* should have comment" + - "^comment on" + - "should be" + + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + # Exclude some linters from running on tests files. + - path: _test\.go + linters: + - gocyclo + - errcheck + - dupl + - gosec + - lll + - path: html_.*\.go + linters: + - lll + - path: model/.*\.go + linters: + - lll + + # Exclude known linters from partially hard-vendored code, + # which is impossible to exclude via "nolint" comments. + - linters: + - gosec + text: "G402" + - linters: + - gosec + text: "G501" + - linters: + - gosec + text: "G404" + - linters: + - gosec + text: "G401" + - linters: + - gosec + text: "G505" + # Exclude some staticcheck messages + - linters: + - staticcheck + text: "SA9003:" + + # Exclude lll issues for long lines with go:generate + - linters: + - lll + source: "^//go:generate " + + # Independently from option `exclude` we use default exclude patterns, + # it can be disabled by this option. To list all + # excluded by default patterns execute `golangci-lint run --help`. + # Default value for this option is true. + exclude-use-default: false + + # The default value is false. If set to true exclude and exclude-rules + # regular expressions become case sensitive. + exclude-case-sensitive: false + + # The list of ids of default excludes to include or disable. By default it's empty. + include: + - EXC0002 # disable excluding of issues about comments from golint + + # Maximum issues count per one linter. Set to 0 to disable. Default is 50. + max-issues-per-linter: 0 + + # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. + max-same-issues: 0 + + # Show only new issues: if there are unstaged changes or untracked files, + # only those changes are analyzed, else only changes in HEAD~ are analyzed. + # It's a super-useful option for integration of golangci-lint into existing + # large codebase. It's not practical to fix all existing issues at the moment + # of integration: much better don't allow issues in new code. + # Default is false. + new: false + + # Show only new issues created after git revision `REV` + #new-from-rev: REV + # Show only new issues created in git patch with set file path. + #new-from-patch: path/to/patch/file + +severity: + # Default value is empty string. + # Set the default severity for issues. If severity rules are defined and the issues + # do not match or no severity is provided to the rule this will be the default + # severity applied. Severities should match the supported severity names of the + # selected out format. + # - Code climate: https://docs.codeclimate.com/docs/issues#issue-severity + # - Checkstyle: https://checkstyle.sourceforge.io/property_types.html#severity + # - Github: https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message + default-severity: error + + # The default value is false. + # If set to true severity-rules regular expressions become case sensitive. + case-sensitive: false + + # Default value is empty list. + # When a list of severity rules are provided, severity information will be added to lint + # issues. Severity rules have the same filtering capability as exclude rules except you + # are allowed to specify one matcher per severity rule. + # Only affects out formats that support setting severity information. + rules: + - linters: + - dupl + severity: info diff --git a/Makefile b/Makefile index 230fe03..e61be28 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ lint: # goimports -local github.com/Laisky -w . gofmt -s -w . go mod tidy - golangci-lint run -E golint,depguard,gocognit,goconst,gofmt,misspell,exportloopref,durationcheck,nilerr #,gosec,lll + golangci-lint run -c .golangci.lint.yml changelog: ./.scripts/generate_changelog.sh diff --git a/cmd/root.go b/cmd/root.go index 0303327..69fc45f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,3 +1,4 @@ +// Package cmd some useful tools for command argument package cmd import ( diff --git a/cmd/tls.go b/cmd/tls.go index 0b5651a..1ed671f 100644 --- a/cmd/tls.go +++ b/cmd/tls.go @@ -61,7 +61,9 @@ func init() { GenTLS.Flags().Duration("duration", 365*24*time.Hour*10, "Duration that certificate is valid for") GenTLS.Flags().Bool("ca", false, "whether this cert should be its own Certificate Authority") GenTLS.Flags().Int("rsa-bits", 2048, "Size of RSA key to generate. Ignored if --ecdsa-curve is set") - GenTLS.Flags().String("ecdsa-curve", "", "ECDSA curve to use to generate a key. Valid values are P224, P256 (recommended), P384, P521") + GenTLS.Flags().String("ecdsa-curve", "", + "ECDSA curve to use to generate a key. "+ + "Valid values are P224, P256 (recommended), P384, P521") GenTLS.Flags().Bool("ed25519", false, "Generate an Ed25519 key") } diff --git a/compressor.go b/compressor.go index c20679c..45aa383 100644 --- a/compressor.go +++ b/compressor.go @@ -224,7 +224,7 @@ func Unzip(src string, dest string) (filenames []string, err error) { if r, err = zip.OpenReader(src); err != nil { return nil, errors.Wrap(err, "open src") } - defer r.Close() + defer func() { _ = r.Close() }() for _, f := range r.File { // Store filename/path for returning and using later on @@ -258,13 +258,13 @@ func Unzip(src string, dest string) (filenames []string, err error) { return nil, errors.Wrapf(err, "open file to write: %s", fpath) } Logger.Debug("create file", zap.String("path", filepath.Dir(fpath))) - defer outFile.Close() + defer func() { _ = outFile.Close() }() rc, err := f.Open() if err != nil { return nil, errors.Wrapf(err, "read src file to write: %s", f.Name) } - defer rc.Close() + defer func() { _ = rc.Close() }() if _, err = io.Copy(outFile, rc); err != nil { return nil, errors.Wrap(err, "copy src to dest") @@ -287,10 +287,10 @@ func ZipFiles(output string, files []string) (err error) { if newZipFile, err = os.Create(output); err != nil { return err } - defer newZipFile.Close() + defer func() { _ = newZipFile.Close() }() zipWriter := zip.NewWriter(newZipFile) - defer zipWriter.Close() + defer func() { _ = zipWriter.Close() }() // Add files to zip for _, file := range files { @@ -319,7 +319,10 @@ func AddFileToZip(zipWriter *zip.Writer, filename, basedir string) error { for _, finfoInDir := range fs { _, childDir := filepath.Split(finfoInDir.Name()) - if err = AddFileToZip(zipWriter, filepath.Join(filename, finfoInDir.Name()), filepath.Join(basedir, finfo.Name())); err != nil { + if err = AddFileToZip(zipWriter, + filepath.Join(filename, finfoInDir.Name()), + filepath.Join(basedir, finfo.Name()), + ); err != nil { return errors.Wrapf(err, "zip sub basedir `%s`", childDir) } } @@ -331,7 +334,7 @@ func AddFileToZip(zipWriter *zip.Writer, filename, basedir string) error { if err != nil { return errors.Wrapf(err, "open file: %s", filename) } - defer fileToZip.Close() + defer func() { _ = fileToZip.Close() }() var header *zip.FileHeader if header, err = zip.FileInfoHeader(finfo); err != nil { diff --git a/counter.go b/counter.go index 06593f2..ec370c2 100644 --- a/counter.go +++ b/counter.go @@ -282,16 +282,13 @@ func (c *ParallelCounter) GetQuote(step int64) (from, to int64) { step = step % c.rotatePoint } - // Logger.Info("try acquire lock", zap.Int64("step", step), zap.Int64("lid", c.lockID)) c.Lock() - // Logger.Info("acquired lock", zap.Int64("step", step), zap.Int64("lid", c.lockID)) from = atomic.LoadInt64(&c.n) to = atomic.AddInt64(&c.n, step) - 1 if c.rotatePoint > 0 && to > c.rotatePoint { // need rotate from, to = 0, step atomic.StoreInt64(&c.n, to+1) } - // Logger.Info("release lock", zap.Int64("step", step), zap.Int64("from", from), zap.Int64("lid", c.lockID), zap.Int64("to", to)) c.Unlock() Logger.Debug("get quote", diff --git a/encrypt.go b/encrypt.go index e0dd7eb..4762f27 100644 --- a/encrypt.go +++ b/encrypt.go @@ -194,28 +194,11 @@ func VerifyReaderByRSAWithSHA256(pubKey *rsa.PublicKey, reader io.Reader, sig [] const ecdsaSignDelimiter = "." -// FormatECDSASign encode es256 signature by hex -// -// Deprecated: replaced by EncodeES256SignByBase6e -var FormatECDSASign = EncodeES256SignByHex - // EncodeES256SignByHex format ecdsa sign to stirng func EncodeES256SignByHex(a, b *big.Int) string { return FormatBig2Hex(a) + ecdsaSignDelimiter + FormatBig2Hex(b) } -// ParseECDSASign encode es256 signature by base64 -// -// Deprecated: replaced by EncodeES256SignByBase64 -func ParseECDSASign(sign string) (a, b *big.Int, ok bool) { - var err error - if a, b, err = DecodeES256SignByHex(sign); err != nil { - return nil, nil, false - } - - return a, b, true -} - // DecodeES256SignByHex parse ecdsa signature string to two *big.Int func DecodeES256SignByHex(sign string) (a, b *big.Int, err error) { ss := strings.Split(sign, ecdsaSignDelimiter) diff --git a/http.go b/http.go index 70ab28b..5ac1a4a 100644 --- a/http.go +++ b/http.go @@ -82,11 +82,6 @@ func WithHTTPClientInsecure(insecure bool) HTTPClientOptFunc { } } -// GetHTTPClient new http client -// -// Deprecated: replaced by NewHTTPClient -var GetHTTPClient = NewHTTPClient - // NewHTTPClient create http client func NewHTTPClient(opts ...HTTPClientOptFunc) (c *http.Client, err error) { opt := &httpClientOption{ @@ -125,7 +120,12 @@ func RequestJSON(method, url string, request *RequestData, resp interface{}) (er } // RequestJSONWithClient request JSON and return JSON with specific client -func RequestJSONWithClient(httpClient *http.Client, method, url string, request *RequestData, resp interface{}) (err error) { +func RequestJSONWithClient(httpClient *http.Client, + method, + url string, + request *RequestData, + resp interface{}, +) (err error) { Logger.Debug("try to request with json", zap.String("method", method), zap.String("url", url)) var ( @@ -147,7 +147,7 @@ func RequestJSONWithClient(httpClient *http.Client, method, url string, request if err != nil { return errors.Wrap(err, "try to request url error") } - defer r.Body.Close() + defer func() { _ = r.Body.Close() }() if r.StatusCode/100 != 2 { respBytes, err := ioutil.ReadAll(r.Body) @@ -197,7 +197,7 @@ func checkRespBody(c *chaining.Chain) (interface{}, error) { return c.GetVal(), nil } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() respB, err := ioutil.ReadAll(resp.Body) if err != nil { return resp, errors.Wrapf(upErr, "read body got error: %v", err.Error()) diff --git a/logger.go b/logger.go index 8bb03eb..bf53c44 100644 --- a/logger.go +++ b/logger.go @@ -426,8 +426,15 @@ func NewAlertPusher(ctx context.Context, pushAPI string, opts ...AlertHookOptFun } // NewAlertPusherWithAlertType create new AlertPusher with default type and token -func NewAlertPusherWithAlertType(ctx context.Context, pushAPI string, alertType, pushToken string, opts ...AlertHookOptFunc) (a *AlertPusher, err error) { - Logger.Debug("create new AlertPusher with alert type", zap.String("pushAPI", pushAPI), zap.String("type", alertType)) +func NewAlertPusherWithAlertType(ctx context.Context, + pushAPI string, + alertType, + pushToken string, + opts ...AlertHookOptFunc, +) (a *AlertPusher, err error) { + Logger.Debug("create new AlertPusher with alert type", + zap.String("pushAPI", pushAPI), + zap.String("type", alertType)) if a, err = NewAlertPusher(ctx, pushAPI, opts...); err != nil { return nil, err } @@ -547,7 +554,11 @@ type PateoAlertPusher struct { } // NewPateoAlertPusher create new PateoAlertPusher -func NewPateoAlertPusher(ctx context.Context, api, token string, opts ...AlertHookOptFunc) (p *PateoAlertPusher, err error) { +func NewPateoAlertPusher(ctx context.Context, + api, + token string, + opts ...AlertHookOptFunc, +) (p *PateoAlertPusher, err error) { opt := newAlertHookOpt() for _, optf := range opts { optf(opt) diff --git a/logger_test.go b/logger_test.go index 154e679..89e3dc6 100644 --- a/logger_test.go +++ b/logger_test.go @@ -246,7 +246,8 @@ func TestChangeLoggerLevel(t *testing.T) { logger.Debug(msg) require.Len(t, allLogs, 1) require.NotEqual(t, msg, allLogs[len(allLogs)-1]) - logger.ChangeLevel(LoggerLevelDebug) + err = logger.ChangeLevel(LoggerLevelDebug) + require.NoError(t, err) } // case: change level for child logger diff --git a/net.go b/net.go index 84fa419..7dc5443 100644 --- a/net.go +++ b/net.go @@ -23,7 +23,7 @@ func IsRemoteUDPPortOpen(addr string) error { if err != nil { return errors.WithStack(err) } - defer conn.Close() + defer func() { _ = conn.Close() }() if err = conn.SetDeadline(Clock.GetUTCNow().Add(3 * time.Second)); err != nil { return errors.WithStack(err) diff --git a/settings.go b/settings.go index e9214fc..9c19a81 100644 --- a/settings.go +++ b/settings.go @@ -137,32 +137,11 @@ func (s *SettingsType) GetStringMapString(key string) map[string]string { return viper.GetStringMapString(key) } -// Setup load config file settings.yml -// -// Deprecated: use LoadFromDir instead -func (s *SettingsType) Setup(configPath string) error { - return s.LoadFromDir(configPath) -} - -// SetupFromDir load settings from dir, default fname is `settings.yml` -// -// Deprecated: use LoadFromDir instead -func (s *SettingsType) SetupFromDir(dirPath string) error { - return s.LoadFromDir(dirPath) -} - // LoadFromDir load settings from dir, default fname is `settings.yml` func (s *SettingsType) LoadFromDir(dirPath string) error { Logger.Info("Setup settings", zap.String("dirpath", dirPath)) fpath := filepath.Join(dirPath, defaultConfigFileName) - return s.SetupFromFile(fpath) -} - -// SetupFromFile load settings from file -// -// Deprecated: use LoadFromFile instead -func (s *SettingsType) SetupFromFile(filePath string) error { - return s.LoadFromFile(filePath) + return s.LoadFromFile(fpath) } type settingsOpt struct { @@ -241,7 +220,7 @@ RECUR_INCLUDE_LOOP: if fp, err = os.Open(filePath); err != nil { return errors.Wrapf(err, "open config file `%s`", filePath) } - defer fp.Close() + defer func() { _ = fp.Close() }() viper.SetConfigType(strings.TrimLeft(filepath.Ext(filePath), ".")) if isSettingsFileEncrypted(opt, filePath) { @@ -292,7 +271,7 @@ func (s *SettingsType) loadConfigFiles(opt *settingsOpt, cfgFiles []string) (err if fp, err = os.Open(filePath); err != nil { return errors.Wrapf(err, "open config file `%s`", filePath) } - defer fp.Close() + defer func() { _ = fp.Close() }() if isSettingsFileEncrypted(opt, filePath) { encryptedFp, err := NewAesReaderWrapper(fp, opt.aesKey) @@ -317,13 +296,6 @@ func (s *SettingsType) loadConfigFiles(opt *settingsOpt, cfgFiles []string) (err return nil } -// SetupFromConfigServer load configs from config-server, -// -// Deprecated: use LoadFromConfigServer instead -func (s *SettingsType) SetupFromConfigServer(url, app, profile, label string) (err error) { - return s.LoadFromConfigServer(url, app, profile, label) -} - // LoadFromConfigServer load configs from config-server, // // endpoint `{url}/{app}/{profile}/{label}` @@ -343,13 +315,6 @@ func (s *SettingsType) LoadFromConfigServer(url, app, profile, label string) (er return nil } -// SetupFromConfigServerWithRawYaml load configs from config-server -// -// Deprecated: use LoadFromConfigServer instead -func (s *SettingsType) SetupFromConfigServerWithRawYaml(url, app, profile, label, key string) (err error) { - return s.LoadFromConfigServerWithRawYaml(url, app, profile, label, key) -} - // LoadFromConfigServerWithRawYaml load configs from config-server // // endpoint `{url}/{app}/{profile}/{label}` diff --git a/settings_test.go b/settings_test.go index c5c1ee0..2f615d6 100644 --- a/settings_test.go +++ b/settings_test.go @@ -22,7 +22,7 @@ func ExampleSettings() { // bind pflags to settings if err := Settings.BindPFlags(pflag.CommandLine); err != nil { - Logger.Panic("parse command") + panic(err) } // use @@ -45,7 +45,7 @@ func ExampleSettings_cobra() { rootCmd := &cobra.Command{} childCmd := &cobra.Command{ PreRun: func(cmd *cobra.Command, args []string) { - if err := Settings.BindPFlags(cmd.Flags()); err != nil { + Settings.BindPFlags(cmd.Flags()); err != nil { Logger.Panic("parse args") } }, @@ -94,7 +94,7 @@ k5: 14 } t.Logf("load settings from: %v", dirName) - if err = Settings.Setup(dirName); err != nil { + if err = Settings.LoadFromDir(dirName); err != nil { t.Fatalf("setup settings got error: %+v", err) } @@ -256,9 +256,9 @@ a: addr := fmt.Sprintf("http://localhost:%v", port) go runMockHTTPServer(ctx, port, "/app/profile/label", fakedata) time.Sleep(100 * time.Millisecond) - if err := Settings.SetupFromConfigServerWithRawYaml(addr, "app", "profile", "label", "raw"); err != nil { - t.Fatalf("got error: %+v", err) - } + err := Settings.LoadFromConfigServerWithRawYaml(addr, "app", "profile", "label", "raw") + require.NoError(t, err) + for k, vi := range map[string]interface{}{ "a.b": 123, "a.c": "abc", @@ -299,7 +299,7 @@ a: } } cfg := &cfgStruct{} - err := Settings.Unmarshal(cfg) + err = Settings.Unmarshal(cfg) require.NoError(t, err) require.Equal(t, uint(123), cfg.A.B) require.Equal(t, "abc", cfg.A.C) diff --git a/sync.go b/sync.go index a44c020..47802a0 100644 --- a/sync.go +++ b/sync.go @@ -9,15 +9,15 @@ import ( "github.com/pkg/errors" ) -const ( - defaultLaiskyRemoteLockTokenUserKey = "uid" - defaultLaiskyRemoteLockAuthCookieName = "general" - defaultLaiskyRemoteLockTimeout = 5 * time.Second - defaultLaiskyRemoteLockRenewalDuration = 10 * time.Second - defaultLaiskyRemoteLockRenewalInterval = 1 * time.Second - defaultLaiskyRemoteLockIsRenewal = false - defaultLaiskyRemoteLockMaxRetry = 3 -) +// const ( +// defaultLaiskyRemoteLockTokenUserKey = "uid" +// defaultLaiskyRemoteLockAuthCookieName = "general" +// defaultLaiskyRemoteLockTimeout = 5 * time.Second +// defaultLaiskyRemoteLockRenewalDuration = 10 * time.Second +// defaultLaiskyRemoteLockRenewalInterval = 1 * time.Second +// defaultLaiskyRemoteLockIsRenewal = false +// defaultLaiskyRemoteLockMaxRetry = 3 +// ) // Mutex mutex that support unblocking lock type Mutex struct { @@ -122,7 +122,10 @@ func (m *Mutex) SpinLock(step, timeout time.Duration) { // } // type acquireLockMutation struct { -// AcquireLock bool `graphql:"AcquireLock(lock_name: $lock_name, is_renewal: $is_renewal, duration_sec: $duration_sec)"` +// AcquireLock bool `graphql:"AcquireLock( +// lock_name: $lock_name, +// is_renewal: $is_renewal, +// duration_sec: $duration_sec)"` // } // type acquireLockOption struct { @@ -182,7 +185,9 @@ func (m *Mutex) SpinLock(step, timeout time.Duration) { // // AcquireLock acquire lock with lockname, // // if `isRenewal=true`, will automate refresh lock's lease until ctx done. // // duration to specify how much time each renewal will extend. -// func (l *LaiskyRemoteLock) AcquireLock(ctx context.Context, lockName string, opts ...AcquireLockOptFunc) (ok bool, err error) { +// func (l *LaiskyRemoteLock) AcquireLock(ctx context.Context, +// lockName string, +// opts ...AcquireLockOptFunc) (ok bool, err error) { // opt := &acquireLockOption{ // renewalInterval: defaultLaiskyRemoteLockRenewalInterval, // duration: defaultLaiskyRemoteLockRenewalDuration, @@ -212,7 +217,9 @@ func (m *Mutex) SpinLock(step, timeout time.Duration) { // return ok, nil // } -// func (l *LaiskyRemoteLock) renewalLock(ctx context.Context, query *acquireLockMutation, vars map[string]interface{}, opt *acquireLockOption) { +// func (l *LaiskyRemoteLock) renewalLock(ctx context.Context, +// query *acquireLockMutation, +// vars map[string]interface{}, opt *acquireLockOption) { // var ( // nRetry = 0 // err error @@ -229,7 +236,9 @@ func (m *Mutex) SpinLock(step, timeout time.Duration) { // } // if err = l.cli.Mutate(ctx, query, vars); err != nil { -// Logger.Error("renewal lock", zap.Error(err), zap.Int("n_retry", nRetry), zap.String("lock_name", lockName)) +// Logger.Error("renewal lock", +// zap.Error(err), +// zap.Int("n_retry", nRetry), zap.String("lock_name", lockName)) // time.Sleep(1 * time.Second) // nRetry++ // continue diff --git a/throttle.go b/throttle.go index 2caaa2b..f63d908 100644 --- a/throttle.go +++ b/throttle.go @@ -90,10 +90,3 @@ TOKEN_LOOP: func (t *Throttle) Close() { close(t.stopChan) } - -// Stop stop throttle -// -// Deprecated: replaced by Close -func (t *Throttle) Stop() { - t.Close() -} diff --git a/throttle_test.go b/throttle_test.go index 5b2c9a5..e656fad 100644 --- a/throttle_test.go +++ b/throttle_test.go @@ -41,7 +41,7 @@ func TestThrottle2(t *testing.T) { Max: 100, }) require.NoError(t, err) - throttle2.Stop() + throttle2.Close() ctx2, cancel := context.WithCancel(ctx) _, err = NewThrottleWithCtx(ctx2, &ThrottleCfg{ diff --git a/time.go b/time.go index 52631f7..4ef5c2c 100644 --- a/time.go +++ b/time.go @@ -88,13 +88,6 @@ type ClockItf interface { const defaultClockInterval = 10 * time.Millisecond -// SetupClock setup internal Clock with step -// -// Deprecated: use SetInternalClock instead -func SetupClock(refreshInterval time.Duration) { - SetInternalClock(refreshInterval) -} - // SetInternalClock set internal Clock with refresh interval func SetInternalClock(interval time.Duration) { if interval < time.Microsecond { @@ -104,7 +97,7 @@ func SetInternalClock(interval time.Duration) { if Clock == nil { Clock = NewClock(context.Background(), interval) } else { - Clock.SetupInterval(interval) + Clock.SetInterval(interval) } } @@ -183,13 +176,6 @@ func (c *ClockType) GetTimeInRFC3339Nano() string { return c.GetUTCNow().Format(time.RFC3339Nano) } -// SetupInterval setup update interval -// -// Deprecated: use SetInterval instead -func (c *ClockType) SetupInterval(interval time.Duration) { - c.SetInterval(interval) -} - // SetInterval setup update interval func (c *ClockType) SetInterval(interval time.Duration) { c.Lock() diff --git a/time_test.go b/time_test.go index 8e3c1b2..2673956 100644 --- a/time_test.go +++ b/time_test.go @@ -94,7 +94,7 @@ func ExampleClock() { Clock.GetTimeInRFC3339Nano() // change clock refresh step - SetupClock(10 * time.Millisecond) + SetInternalClock(10 * time.Millisecond) // create new clock c := NewClock(context.Background(), 1*time.Second) @@ -108,7 +108,6 @@ func TestClock2(t *testing.T) { var err error t.Logf("ts: %v", ts.Format(time.RFC3339Nano)) - c.SetupInterval(100 * time.Millisecond) c.SetInterval(100 * time.Millisecond) // test ts @@ -271,43 +270,43 @@ func BenchmarkClock(b *testing.B) { clock2.GetUTCNow() } }) - clock2.SetupInterval(100 * time.Millisecond) + clock2.SetInterval(100 * time.Millisecond) b.Run("clock2 time with 100ms", func(b *testing.B) { for i := 0; i < b.N; i++ { clock2.GetUTCNow() } }) - clock2.SetupInterval(10 * time.Millisecond) + clock2.SetInterval(10 * time.Millisecond) b.Run("clock2 time with 10ms", func(b *testing.B) { for i := 0; i < b.N; i++ { clock2.GetUTCNow() } }) - clock2.SetupInterval(1 * time.Millisecond) + clock2.SetInterval(1 * time.Millisecond) b.Run("clock2 time with 1ms", func(b *testing.B) { for i := 0; i < b.N; i++ { clock2.GetUTCNow() } }) - clock2.SetupInterval(500 * time.Microsecond) + clock2.SetInterval(500 * time.Microsecond) b.Run("clock2 time with 500us", func(b *testing.B) { for i := 0; i < b.N; i++ { clock2.GetUTCNow() } }) - clock2.SetupInterval(100 * time.Microsecond) + clock2.SetInterval(100 * time.Microsecond) b.Run("clock2 time with 100us", func(b *testing.B) { for i := 0; i < b.N; i++ { clock2.GetUTCNow() } }) - clock2.SetupInterval(10 * time.Microsecond) + clock2.SetInterval(10 * time.Microsecond) b.Run("clock2 time with 10us", func(b *testing.B) { for i := 0; i < b.N; i++ { clock2.GetUTCNow() } }) - clock2.SetupInterval(1 * time.Microsecond) + clock2.SetInterval(1 * time.Microsecond) b.Run("clock2 time with 10us", func(b *testing.B) { for i := 0; i < b.N; i++ { clock2.GetUTCNow() @@ -316,7 +315,6 @@ func BenchmarkClock(b *testing.B) { } func TestSetupClock(t *testing.T) { - SetupClock(100 * time.Millisecond) SetInternalClock(100 * time.Millisecond) // case: invalid interval diff --git a/utils.go b/utils.go index f1db0a1..ce9e174 100644 --- a/utils.go +++ b/utils.go @@ -132,7 +132,7 @@ func ValidateFileHash(filepath string, hashed string) error { if err != nil { return errors.Wrapf(err, "open file `%s`", filepath) } - defer fp.Close() + defer func() { _ = fp.Close() }() if _, err = io.Copy(hasher, fp); err != nil { return errors.Wrap(err, "read file content") @@ -268,7 +268,7 @@ func AutoGC(ctx context.Context, opts ...GcOptFunc) (err error) { if fp, err = os.Open(opt.memLimitFilePath); err != nil { return errors.Wrapf(err, "open file got error: %+v", opt.memLimitFilePath) } - defer fp.Close() + defer func() { _ = fp.Close() }() if memByte, err = ioutil.ReadAll(fp); err != nil { return errors.Wrap(err, "read cgroup mem limit file") } From 783380bdfccb3da3dbf5f2e6601eb6a48d024ed8 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Tue, 7 Sep 2021 06:10:25 +0000 Subject: [PATCH 13/14] refactor: rename --- sync.go | 4 ++-- utils.go | 34 ++++++++++++++++++---------------- utils_test.go | 8 ++++---- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/sync.go b/sync.go index 47802a0..203cf16 100644 --- a/sync.go +++ b/sync.go @@ -250,13 +250,13 @@ func (m *Mutex) SpinLock(step, timeout time.Duration) { // ExpiredRLock Lock with expire time type ExpiredRLock struct { - m *ExpiredMap + m *LRUExpiredMap } // NewExpiredRLock new ExpiredRLock func NewExpiredRLock(ctx context.Context, exp time.Duration) (el *ExpiredRLock, err error) { el = &ExpiredRLock{} - el.m, err = NewExpiredMap(ctx, exp, func() interface{} { + el.m, err = NewLRUExpiredMap(ctx, exp, func() interface{} { return &sync.RWMutex{} }) err = errors.Wrap(err, "new expired rlock") diff --git a/utils.go b/utils.go index ce9e174..79cbbae 100644 --- a/utils.go +++ b/utils.go @@ -497,23 +497,23 @@ func Base64Decode(encoded string) ([]byte, error) { return base64.URLEncoding.DecodeString(encoded) } -// SimpleExpCache single item with expires -type SimpleExpCache struct { +// SingleItemExpCache single item with expires +type SingleItemExpCache struct { expiredAt time.Time ttl time.Duration data interface{} mu sync.RWMutex } -// NewSimpleExpCache new expcache contains single data -func NewSimpleExpCache(ttl time.Duration) *SimpleExpCache { - return &SimpleExpCache{ +// NewSingleItemExpCache new expcache contains single data +func NewSingleItemExpCache(ttl time.Duration) *SingleItemExpCache { + return &SingleItemExpCache{ ttl: ttl, } } // Set set data and refresh expires -func (c *SimpleExpCache) Set(data interface{}) { +func (c *SingleItemExpCache) Set(data interface{}) { c.mu.Lock() c.data = data c.expiredAt = Clock.GetUTCNow().Add(c.ttl) @@ -523,7 +523,7 @@ func (c *SimpleExpCache) Set(data interface{}) { // Get get data // // if data is expired, ok=false -func (c *SimpleExpCache) Get() (data interface{}, ok bool) { +func (c *SingleItemExpCache) Get() (data interface{}, ok bool) { c.mu.RLock() data = c.data @@ -534,7 +534,7 @@ func (c *SimpleExpCache) Get() (data interface{}, ok bool) { } // GetString same as Get, but return string -func (c *SimpleExpCache) GetString() (data string, ok bool) { +func (c *SingleItemExpCache) GetString() (data string, ok bool) { var itf interface{} if itf, ok = c.Get(); !ok { return "", false @@ -544,7 +544,7 @@ func (c *SimpleExpCache) GetString() (data string, ok bool) { } // GetUintSlice same as Get, but return []uint -func (c *SimpleExpCache) GetUintSlice() (data []uint, ok bool) { +func (c *SingleItemExpCache) GetUintSlice() (data []uint, ok bool) { var itf interface{} if itf, ok = c.Get(); !ok { return nil, false @@ -634,18 +634,20 @@ func (e *expiredMapItem) refreshTime() { atomic.StoreInt64(e.t, Clock.GetUTCNow().Unix()) } -// ExpiredMap map with expire time, auto delete expired item. +// LRUExpiredMap map with expire time, auto delete expired item. // // `Get` will auto refresh item's expires. -type ExpiredMap struct { +type LRUExpiredMap struct { m sync.Map ttl time.Duration new func() interface{} } -// NewExpiredMap new ExpiredMap -func NewExpiredMap(ctx context.Context, ttl time.Duration, new func() interface{}) (el *ExpiredMap, err error) { - el = &ExpiredMap{ +// NewLRUExpiredMap new ExpiredMap +func NewLRUExpiredMap(ctx context.Context, + ttl time.Duration, + new func() interface{}) (el *LRUExpiredMap, err error) { + el = &LRUExpiredMap{ ttl: ttl, new: new, } @@ -654,7 +656,7 @@ func NewExpiredMap(ctx context.Context, ttl time.Duration, new func() interface{ return el, nil } -func (e *ExpiredMap) clean(ctx context.Context) { +func (e *LRUExpiredMap) clean(ctx context.Context) { for { select { case <-ctx.Done(): @@ -686,7 +688,7 @@ func (e *ExpiredMap) clean(ctx context.Context) { // Get get item // // will auto refresh key's ttl -func (e *ExpiredMap) Get(key string) interface{} { +func (e *LRUExpiredMap) Get(key string) interface{} { l, _ := e.m.Load(key) if l == nil { t := Clock.GetUTCNow().Unix() diff --git a/utils_test.go b/utils_test.go index 2a3a369..a6b6b13 100644 --- a/utils_test.go +++ b/utils_test.go @@ -694,7 +694,7 @@ func TestExpCache_Store(t *testing.T) { // PASS // ok github.com/Laisky/go-utils 1.573s func BenchmarkExpMap(b *testing.B) { - cm, err := NewExpiredMap(context.Background(), + cm, err := NewLRUExpiredMap(context.Background(), 10*time.Millisecond, func() interface{} { return 1 }, ) @@ -755,7 +755,7 @@ func TestGetStructFieldByName(t *testing.T) { } func Benchmark_NewSimpleExpCache(b *testing.B) { - c := NewSimpleExpCache(time.Millisecond) + c := NewSingleItemExpCache(time.Millisecond) b.RunParallel(func(pb *testing.PB) { for pb.Next() { if rand.Intn(10) < 5 { @@ -774,7 +774,7 @@ func TestNewSimpleExpCache(t *testing.T) { // time.clock's test set interval to 100ms. fmt.Println("interval", Clock.Interval()) Clock.SetInterval(1 * time.Microsecond) - c := NewSimpleExpCache(200 * time.Millisecond) + c := NewSingleItemExpCache(200 * time.Millisecond) _, ok := c.Get() require.False(t, ok) @@ -801,7 +801,7 @@ func TestNewSimpleExpCache(t *testing.T) { func TestNewExpiredMap(t *testing.T) { ctx := context.Background() - m, err := NewExpiredMap(ctx, time.Millisecond, func() interface{} { return 666 }) + m, err := NewLRUExpiredMap(ctx, time.Millisecond, func() interface{} { return 666 }) require.NoError(t, err) const key = "key" From e0998de1f6f0f7b71f6a84556012f5f0c18f07a2 Mon Sep 17 00:00:00 2001 From: "Laisky.Cai" Date: Fri, 10 Sep 2021 06:59:14 +0000 Subject: [PATCH 14/14] feat: add `Delete` to ExpCache --- go.mod | 4 +--- go.sum | 4 ++-- skiplist.go | 2 +- utils.go | 5 +++++ 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5ff882b..da651c3 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/Laisky/go-utils go 1.13 require ( + github.com/Laisky/fast-skiplist v0.0.0-20210907063351-e00546c800a6 github.com/Laisky/go-chaining v0.0.0-20180507092046-43dcdc5a21be github.com/Laisky/graphql v1.0.5 github.com/Laisky/zap v1.12.3-0.20210804015521-853b5a8ec429 @@ -22,7 +23,4 @@ require ( golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df - skiplist v0.0.0-00010101000000-000000000000 ) - -replace skiplist => github.com/Laisky/fast-skiplist v0.0.0-20210903093518-aea9c86c0252 diff --git a/go.sum b/go.sum index e44ac45..cb1b915 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Laisky/fast-skiplist v0.0.0-20210903093518-aea9c86c0252 h1:5MGOoFlNUR7PX3ppBzzg5nSxy4eDFpWaZZYAfb9ktKk= -github.com/Laisky/fast-skiplist v0.0.0-20210903093518-aea9c86c0252/go.mod h1:1qPOW+o/q+5G/cjALu4aG3CKmwVomSwqlL5FnoJanOE= +github.com/Laisky/fast-skiplist v0.0.0-20210907063351-e00546c800a6 h1:V3MNilKNjNVKwBiMLsDCa6dFbA5gROE7TYxJJBXkwTc= +github.com/Laisky/fast-skiplist v0.0.0-20210907063351-e00546c800a6/go.mod h1:BdpS7FU5kzp0QV/dtCWheAV5rIOQu+Jb+uoVuVWZOvU= github.com/Laisky/go-chaining v0.0.0-20180507092046-43dcdc5a21be h1:7Rxhm6IjOtDAyj8eScOFntevwzkWhx94zi48lxo4m4w= github.com/Laisky/go-chaining v0.0.0-20180507092046-43dcdc5a21be/go.mod h1:1mdzaETo0kjvCQICPSePsoaatJN4l7JvEA1200lyevo= github.com/Laisky/graphql v1.0.5 h1:8eJ7mrXKVkKxZ+Nw1HPs3iQPVNxXGctysqTEY0lNBlc= diff --git a/skiplist.go b/skiplist.go index c76ac00..bbe7889 100644 --- a/skiplist.go +++ b/skiplist.go @@ -1,6 +1,6 @@ package utils -import "skiplist" +import skiplist "github.com/Laisky/fast-skiplist" func NewSkiplist() *skiplist.SkipList { return skiplist.New() diff --git a/utils.go b/utils.go index 79cbbae..2d71dd6 100644 --- a/utils.go +++ b/utils.go @@ -608,6 +608,11 @@ func (c *ExpCache) Store(key, val interface{}) { }) } +// Delete remove key +func (c *ExpCache) Delete(key interface{}) { + c.data.Delete(key) +} + // Load load val from cache func (c *ExpCache) Load(key interface{}) (data interface{}, ok bool) { if data, ok = c.data.Load(key); ok && Clock.GetUTCNow().Before(data.(*expCacheItem).exp) {