diff --git a/go.mod b/go.mod index 3471214f..f8f9400c 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/streadway/amqp v0.0.0-20170521212453-dfe15e360485 github.com/stretchr/testify v1.8.4 - github.com/taylorchu/toki v0.0.0-20141019163204-20e86122596c github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c ) diff --git a/go.sum b/go.sum index 674cf9ea..8f92572a 100644 --- a/go.sum +++ b/go.sum @@ -107,8 +107,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/taylorchu/toki v0.0.0-20141019163204-20e86122596c h1:FPVNYOiTjaGNfDYevBEiLLu9iSkBdvnfMcnR6qKkwbg= -github.com/taylorchu/toki v0.0.0-20141019163204-20e86122596c/go.mod h1:YYyjUTaSaC2W8KSnbqKGHDKPd/e/8c88su0LP0NkUfk= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= diff --git a/imperatives/imperatives.go b/imperatives/imperatives.go index e51117b9..32bbedb8 100644 --- a/imperatives/imperatives.go +++ b/imperatives/imperatives.go @@ -14,12 +14,12 @@ import ( "github.com/grafana/carbon-relay-ng/rewriter" "github.com/grafana/carbon-relay-ng/route" "github.com/grafana/carbon-relay-ng/table" + "github.com/grafana/carbon-relay-ng/tokre" "github.com/grafana/metrictank/cluster/partitioner" - "github.com/taylorchu/toki" ) const ( - addBlack toki.Token = iota + addBlack tokre.Token = iota addBlock addAgg addRouteSendAllMatch @@ -99,7 +99,7 @@ const ( // we should make sure we apply changes atomatically. e.g. when changing dest between address A and pickle=false and B with pickle=true, // we should never half apply the change if one of them fails. -var tokens = []toki.Def{ +var tokens = []tokre.Def{ {Token: addBlack, Pattern: "addBlack"}, {Token: addBlock, Pattern: "addBlock"}, {Token: addAgg, Pattern: "addAgg"}, @@ -192,7 +192,7 @@ var errFmtModRoute = errors.New("modRoute ") var errOrgId0 = errors.New("orgId must be a number > 0") func Apply(table table.Interface, cmd string) error { - s := toki.NewScanner(tokens) + s := tokre.NewScanner(tokens) s.SetInput(strings.Replace(cmd, " ", " ## ", -1)) // token library skips whitespace but for us double space is significant t := s.Next() switch t.Token { @@ -229,7 +229,7 @@ func Apply(table table.Interface, cmd string) error { } } -func readAddAgg(s *toki.Scanner, table table.Interface) error { +func readAddAgg(s *tokre.Scanner, table table.Interface) error { t := s.Next() if t.Token != sumFn && t.Token != avgFn && t.Token != minFn && t.Token != maxFn && t.Token != lastFn && t.Token != deltaFn && t.Token != countFn && t.Token != deriveFn && t.Token != stdevFn { return errors.New("invalid function. need avg/max/min/sum/last/count/delta/derive/stdev") @@ -250,7 +250,7 @@ func readAddAgg(s *toki.Scanner, table table.Interface) error { t = s.Next() } // scan for prefix/sub/regex=, stop when we hit a bare word (outFmt) - for ; t.Token != toki.EOF && t.Token != word; t = s.Next() { + for ; t.Token != tokre.EOF && t.Token != word; t = s.Next() { switch t.Token { case optPrefix: if t = s.Next(); t.Token != word { @@ -316,7 +316,7 @@ func readAddAgg(s *toki.Scanner, table table.Interface) error { dropRaw := false t = s.Next() - for ; t.Token != toki.EOF; t = s.Next() { + for ; t.Token != tokre.EOF; t = s.Next() { switch t.Token { case optCache: t = s.Next() @@ -356,7 +356,7 @@ func readAddAgg(s *toki.Scanner, table table.Interface) error { return nil } -func readAddBlock(s *toki.Scanner, table table.Interface) error { +func readAddBlock(s *tokre.Scanner, table table.Interface) error { prefix := "" notPrefix := "" sub := "" @@ -411,7 +411,7 @@ func readAddBlock(s *toki.Scanner, table table.Interface) error { return nil } -func readAddRoute(s *toki.Scanner, table table.Interface, constructor func(key string, matcher matcher.Matcher, destinations []*destination.Destination) (route.Route, error)) error { +func readAddRoute(s *tokre.Scanner, table table.Interface, constructor func(key string, matcher matcher.Matcher, destinations []*destination.Destination) (route.Route, error)) error { t := s.Next() if t.Token != word { return errFmtAddRoute @@ -444,7 +444,7 @@ func readAddRoute(s *toki.Scanner, table table.Interface, constructor func(key s return nil } -func readAddRouteConsistentHashing(s *toki.Scanner, table table.Interface, withFix bool) error { +func readAddRouteConsistentHashing(s *tokre.Scanner, table table.Interface, withFix bool) error { t := s.Next() if t.Token != word { return errFmtAddRoute @@ -476,7 +476,7 @@ func readAddRouteConsistentHashing(s *toki.Scanner, table table.Interface, withF table.AddRoute(route) return nil } -func readAddRouteGrafanaNet(s *toki.Scanner, table table.Interface) error { +func readAddRouteGrafanaNet(s *tokre.Scanner, table table.Interface) error { t := s.Next() if t.Token != word { return errFmtAddRouteGrafanaNet @@ -519,7 +519,7 @@ func readAddRouteGrafanaNet(s *toki.Scanner, table table.Interface) error { t = s.Next() - for ; t.Token != toki.EOF; t = s.Next() { + for ; t.Token != tokre.EOF; t = s.Next() { switch t.Token { case optAggregationFile: t = s.Next() @@ -663,7 +663,7 @@ func readAddRouteGrafanaNet(s *toki.Scanner, table table.Interface) error { return nil } -func readAddRouteKafkaMdm(s *toki.Scanner, table table.Interface) error { +func readAddRouteKafkaMdm(s *tokre.Scanner, table table.Interface) error { t := s.Next() if t.Token != word { return errFmtAddRouteKafkaMdm @@ -739,7 +739,7 @@ func readAddRouteKafkaMdm(s *toki.Scanner, table table.Interface) error { var saslMechanism, saslUsername, saslPassword string t = s.Next() - for ; t.Token != toki.EOF; t = s.Next() { + for ; t.Token != tokre.EOF; t = s.Next() { switch t.Token { case optBlocking: t = s.Next() @@ -864,7 +864,7 @@ func readAddRouteKafkaMdm(s *toki.Scanner, table table.Interface) error { return nil } -func readAddRoutePubSub(s *toki.Scanner, table table.Interface) error { +func readAddRoutePubSub(s *tokre.Scanner, table table.Interface) error { t := s.Next() if t.Token != word { return errFmtAddRoutePubSub @@ -900,7 +900,7 @@ func readAddRoutePubSub(s *toki.Scanner, table table.Interface) error { var blocking = false t = s.Next() - for ; t.Token != toki.EOF; t = s.Next() { + for ; t.Token != tokre.EOF; t = s.Next() { switch t.Token { case optPubSubCodec: t = s.Next() @@ -973,8 +973,8 @@ func readAddRoutePubSub(s *toki.Scanner, table table.Interface) error { return nil } -func readAddRewriter(s *toki.Scanner, table table.Interface) error { - var t *toki.Result +func readAddRewriter(s *tokre.Scanner, table table.Interface) error { + var t tokre.Result if t = s.Next(); t.Token != word { return errFmtAddRewriter } @@ -1002,7 +1002,7 @@ func readAddRewriter(s *toki.Scanner, table table.Interface) error { return nil } -func readDelRoute(s *toki.Scanner, table table.Interface) error { +func readDelRoute(s *tokre.Scanner, table table.Interface) error { t := s.Next() if t.Token != word { return errors.New("need route key") @@ -1011,7 +1011,7 @@ func readDelRoute(s *toki.Scanner, table table.Interface) error { return table.DelRoute(key) } -func readModDest(s *toki.Scanner, table table.Interface) error { +func readModDest(s *tokre.Scanner, table table.Interface) error { t := s.Next() if t.Token != word { return errFmtAddRoute @@ -1028,10 +1028,10 @@ func readModDest(s *toki.Scanner, table table.Interface) error { } opts := make(map[string]string) - for t.Token != toki.EOF { + for t.Token != tokre.EOF { t = s.Next() switch t.Token { - case toki.EOF: + case tokre.EOF: break case optAddr: if t = s.Next(); t.Token != word { @@ -1079,7 +1079,7 @@ func readModDest(s *toki.Scanner, table table.Interface) error { return table.UpdateDestination(key, index, opts) } -func readModRoute(s *toki.Scanner, table table.Interface) error { +func readModRoute(s *tokre.Scanner, table table.Interface) error { t := s.Next() if t.Token != word { return errFmtAddRoute @@ -1087,10 +1087,10 @@ func readModRoute(s *toki.Scanner, table table.Interface) error { key := string(t.Value) opts := make(map[string]string) - for t.Token != toki.EOF { + for t.Token != tokre.EOF { t = s.Next() switch t.Token { - case toki.EOF: + case tokre.EOF: break case optPrefix: if t = s.Next(); t.Token != word { @@ -1136,14 +1136,14 @@ func readModRoute(s *toki.Scanner, table table.Interface) error { // we should read and apply all destinations at once, // or at least make sure we apply them to the global datastruct at once, // otherwise we can destabilize things / send wrong traffic, etc -func readDestinations(s *toki.Scanner, table table.Interface, allowMatcher bool, routeKey string) (destinations []*destination.Destination, err error) { +func readDestinations(s *tokre.Scanner, table table.Interface, allowMatcher bool, routeKey string) (destinations []*destination.Destination, err error) { t := s.Peek() - for t.Token != toki.EOF { + for t.Token != tokre.EOF { for t.Token == sep { s.Next() t = s.Peek() } - if t.Token == toki.EOF { + if t.Token == tokre.EOF { break } @@ -1158,7 +1158,7 @@ func readDestinations(s *toki.Scanner, table table.Interface, allowMatcher bool, return destinations, nil } -func readDestination(s *toki.Scanner, table table.Interface, allowMatcher bool, routeKey string) (dest *destination.Destination, err error) { +func readDestination(s *tokre.Scanner, table table.Interface, allowMatcher bool, routeKey string) (dest *destination.Destination, err error) { var prefix, notPrefix, sub, notSub, regex, notRegex, addr, spoolDir string var spool, pickle bool flush := 1000 @@ -1180,7 +1180,7 @@ func readDestination(s *toki.Scanner, table table.Interface, allowMatcher bool, } addr = string(t.Value) - for t.Token != toki.EOF && t.Token != sep { + for t.Token != tokre.EOF && t.Token != sep { t = s.Next() switch t.Token { case optPrefix: @@ -1314,7 +1314,7 @@ func readDestination(s *toki.Scanner, table table.Interface, allowMatcher bool, return nil, err } unspoolSleep = time.Duration(tmp) * time.Microsecond - case toki.EOF: + case tokre.EOF: case sep: break default: @@ -1336,7 +1336,7 @@ func readDestination(s *toki.Scanner, table table.Interface, allowMatcher bool, } func ParseDestinations(destinationConfigs []string, table table.Interface, allowMatcher bool, routeKey string) (destinations []*destination.Destination, err error) { - s := toki.NewScanner(tokens) + s := tokre.NewScanner(tokens) for _, destinationConfig := range destinationConfigs { s.SetInput(destinationConfig) @@ -1349,13 +1349,13 @@ func ParseDestinations(destinationConfigs []string, table table.Interface, allow return destinations, nil } -func readRouteOpts(s *toki.Scanner) (prefix, notPrefix, sub, notSub, regex, notRegex string, err error) { +func readRouteOpts(s *tokre.Scanner) (prefix, notPrefix, sub, notSub, regex, notRegex string, err error) { for { t := s.Next() switch t.Token { - case toki.EOF: + case tokre.EOF: return - case toki.Error: + case tokre.Error: return "", "", "", "", "", "", errors.New("read the error token instead of one i recognize") case optPrefix: if t = s.Next(); t.Token != word { diff --git a/imperatives/imperatives_test.go b/imperatives/imperatives_test.go index ebdabe18..25e8169a 100644 --- a/imperatives/imperatives_test.go +++ b/imperatives/imperatives_test.go @@ -10,53 +10,53 @@ import ( "github.com/grafana/carbon-relay-ng/pkg/test" "github.com/grafana/carbon-relay-ng/route" "github.com/grafana/carbon-relay-ng/table" - "github.com/taylorchu/toki" + "github.com/grafana/carbon-relay-ng/tokre" ) func TestScanner(t *testing.T) { cases := []struct { cmd string - exp []toki.Token + exp []tokre.Token }{ { "addBlock prefix collectd.localhost", - []toki.Token{addBlock, word, word}, + []tokre.Token{addBlock, word, word}, }, { `addBlock regex ^foo\..*\.cpu+`, - []toki.Token{addBlock, word, word}, + []tokre.Token{addBlock, word, word}, }, { `addAgg sum ^stats\.timers\.(app|proxy|static)[0-9]+\.requests\.(.*) stats.timers._sum_$1.requests.$2 10 20`, - []toki.Token{addAgg, sumFn, word, word, num, num}, + []tokre.Token{addAgg, sumFn, word, word, num, num}, }, { `addAgg avg ^stats\.timers\.(app|proxy|static)[0-9]+\.requests\.(.*) stats.timers._avg_$1.requests.$2 5 10`, - []toki.Token{addAgg, avgFn, word, word, num, num}, + []tokre.Token{addAgg, avgFn, word, word, num, num}, }, { "addRoute sendAllMatch carbon-default 127.0.0.1:2005 spool=true pickle=false", - []toki.Token{addRouteSendAllMatch, word, sep, word, optSpool, optTrue, optPickle, optFalse}, + []tokre.Token{addRouteSendAllMatch, word, sep, word, optSpool, optTrue, optPickle, optFalse}, }, { "addRoute sendAllMatch carbon-tagger sub== 127.0.0.1:2006", - []toki.Token{addRouteSendAllMatch, word, optSub, word, sep, word}, + []tokre.Token{addRouteSendAllMatch, word, optSub, word, sep, word}, }, { "addRoute sendFirstMatch analytics regex=(Err/s|wait_time|logger) graphite.prod:2003 prefix=prod. spool=true pickle=true graphite.staging:2003 prefix=staging. spool=true pickle=true", - []toki.Token{addRouteSendFirstMatch, word, optRegex, word, sep, word, optPrefix, word, optSpool, optTrue, optPickle, optTrue, sep, word, optPrefix, word, optSpool, optTrue, optPickle, optTrue}, + []tokre.Token{addRouteSendFirstMatch, word, optRegex, word, sep, word, optPrefix, word, optSpool, optTrue, optPickle, optTrue, sep, word, optPrefix, word, optSpool, optTrue, optPickle, optTrue}, }, { "addRoute sendFirstMatch myRoute1 127.0.0.1:2003 notPrefix=aaa notSub=bbb notRegex=ccc", - []toki.Token{addRouteSendFirstMatch, word, sep, word, optNotPrefix, word, optNotSub, word, optNotRegex, word}, + []tokre.Token{addRouteSendFirstMatch, word, sep, word, optNotPrefix, word, optNotSub, word, optNotRegex, word}, }, //{ disabled cause tries to read the schemas.conf file // "addRoute grafanaNet grafanaNet http://localhost:8081/metrics your-grafana.net-api-key /path/to/storage-schemas.conf", - // []toki.Token{addRouteGrafanaNet, word, sep, word, word}, + // []tokre.Token{addRouteGrafanaNet, word, sep, word, word}, //}, } for i, c := range cases { - s := toki.NewScanner(tokens) + s := tokre.NewScanner(tokens) s.SetInput(strings.Replace(c.cmd, " ", " ## ", -1)) for j, e := range c.exp { r := s.Next() diff --git a/tokre/README.md b/tokre/README.md new file mode 100644 index 00000000..fb415be5 --- /dev/null +++ b/tokre/README.md @@ -0,0 +1,6 @@ +# Tokre + +Tokenizer based on regular expressions. +Built to be super simple. +Caller provides a list of possible +tokens and the regex definitions. diff --git a/tokre/tokre.go b/tokre/tokre.go new file mode 100644 index 00000000..f3fe92ea --- /dev/null +++ b/tokre/tokre.go @@ -0,0 +1,121 @@ +// Package tokre tokenizes based on regular expressions. +package tokre + +import ( + "fmt" + "regexp" + "strings" +) + +type ( + // Classifies the type of token encountered. + Token int + + // Defines a token type, with the Pattern being a regex. + Def struct { + Token Token + Pattern string + regex *regexp.Regexp + } + + // The position within the input in characters. + // Starting at (1,1). Note that multi-byte sequences must first + // be converted to characters prior to use with this tokenizer. + Position struct { + Line int + Column int + } + + // Created by NewScanner, this is the central place doing the tokenization. + Scanner struct { + defs []Def + input string + pos Position + space_re *regexp.Regexp + } + + // The resulting tokens. + // Each is from scanner.Next, and includes both the position and text found. + Result struct { + Token Token + Pos Position + Value string + } +) + +// Represents the end of input. +const EOF Token = -1 + +// Represents some sort of error occurred, the input could not be matched to any token def. +const Error Token = -2 + +// Create a scanner tokenizer. +func NewScanner(defs []Def) *Scanner { + for i := range defs { + defs[i].regex = regexp.MustCompile("^" + defs[i].Pattern) + } + return &Scanner{ + defs: defs, + space_re: regexp.MustCompile(`^\s+`), + } +} + +// Start the tokenizer with an input string. +func (s *Scanner) SetInput(input string) { + s.input = input + s.pos = Position{1, 1} +} + +// Get the next token from the input. +func (s *Scanner) Next() Result { + s.skip_space() + tok := s.match() + s.consume(tok.Value) + return tok +} + +func (s *Scanner) Peek() Result { + s.skip_space() + return s.match() +} + +// Pretty printable Result. +func (r Result) String() string { + return fmt.Sprintf("(Ln %d, Col %d): %d \"%s\"", r.Pos.Line, r.Pos.Column, r.Token, r.Value) +} + +// Consumes input, updating the state of the tokenizer. +func (s *Scanner) consume(text string) { + s.pos.Line += strings.Count(text, "\n") + last := strings.LastIndex(text, "\n") + if last != -1 { + s.pos.Column = 1 + } + s.pos.Column += len(text) - last - 1 + + s.input = strings.TrimPrefix(s.input, text) +} + +// Skip spaces. +func (s *Scanner) skip_space() { + result := s.space_re.FindString(s.input) + if result == "" { + return + } + s.consume(result) +} + +// Match against the first Def that is at the head of the input. +func (s *Scanner) match() Result { + if len(s.input) == 0 { + return Result{Token: EOF, Pos: s.pos} + } + for _, r := range s.defs { + result := r.regex.FindStringIndex(s.input) + if result == nil { + continue + } + return Result{Token: r.Token, Pos: s.pos, Value: s.input[result[0]:result[1]]} + } + return Result{Token: Error, Pos: s.pos} +} diff --git a/tokre/tokre_test.go b/tokre/tokre_test.go new file mode 100644 index 00000000..2f073d5b --- /dev/null +++ b/tokre/tokre_test.go @@ -0,0 +1,42 @@ +package tokre + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + NUMBER Token = iota + 1 + DASH + STRING + EXCLAMATION +) + +func TestToker(t *testing.T) { + input := "3 - 2-1 - blast off!!!" + + scanner := NewScanner([]Def{ + {Token: NUMBER, Pattern: "[0-9]+"}, + {Token: DASH, Pattern: `-`}, + {Token: STRING, Pattern: "[a-z]+"}, + {Token: EXCLAMATION, Pattern: "!+"}, + }) + + scanner.SetInput(input) + + expected := []Result{ + {NUMBER, Position{1, 1}, "3"}, {DASH, Position{1, 3}, "-"}, + {NUMBER, Position{1, 6}, "2"}, {DASH, Position{1, 7}, "-"}, + {NUMBER, Position{1, 8}, "1"}, {DASH, Position{1, 10}, "-"}, + {STRING, Position{1, 12}, "blast"}, {STRING, Position{1, 18}, "off"}, + {EXCLAMATION, Position{1, 21}, "!!!"}, {EOF, Position{1, 24}, ""}, + } + for _, exp := range expected { + token := scanner.Next() + assert.Equal(t, token.Token, exp.Token) + assert.Equal(t, token.Pos, exp.Pos) + assert.Equal(t, token.Value, exp.Value) + //t.Logf("%s", token) + } +}