-
Notifications
You must be signed in to change notification settings - Fork 64
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add logical planer and select caching
This commit adds the first version of a logical planner with the capability to unify matchers between different selectors. The MergeSelectsOptimizer traverses the AST and identifies the most selective matcher for each individual metric. It then replaces less selective matchers with the most selective one, and adds an additional filters to ensure correctness. The physical plan can then cache results for identical selectors which leads to fewer network calls and faster series retrieval operations.
- Loading branch information
1 parent
700c5ba
commit fac1a24
Showing
18 changed files
with
665 additions
and
133 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| package logicalplan | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| "github.com/prometheus/prometheus/model/labels" | ||
| "github.com/prometheus/prometheus/promql/parser" | ||
| ) | ||
|
|
||
| type FilteredSelector struct { | ||
| *parser.VectorSelector | ||
| Filters []*labels.Matcher | ||
| } | ||
|
|
||
| func (f FilteredSelector) String() string { | ||
| return fmt.Sprintf("filter(%s, %s)", f.Filters, f.VectorSelector.String()) | ||
| } | ||
|
|
||
| func (f FilteredSelector) Pretty(level int) string { return f.String() } | ||
|
|
||
| func (f FilteredSelector) PositionRange() parser.PositionRange { return parser.PositionRange{} } | ||
|
|
||
| func (f FilteredSelector) Type() parser.ValueType { return parser.ValueTypeVector } | ||
|
|
||
| func (f FilteredSelector) PromQLExpr() {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| package logicalplan | ||
|
|
||
| import ( | ||
| "github.com/prometheus/prometheus/model/labels" | ||
| "github.com/prometheus/prometheus/promql/parser" | ||
| ) | ||
|
|
||
| // MergeSelectsOptimizer optimizes a binary expression where | ||
| // one select is a superset of the other select. | ||
| // For example, the expression: | ||
| // metric{a="b", c="d"} / scalar(metric{a="b"}) becomes: | ||
| // Filter(c="d", metric{a="b"}) / scalar(metric{a="b"}). | ||
| // The engine can then cache the result of `metric{a="b"}` | ||
| // and apply an additional filter for {c="d"}. | ||
| type MergeSelectsOptimizer struct{} | ||
|
|
||
| func (m MergeSelectsOptimizer) Optimize(expr parser.Expr) parser.Expr { | ||
| heap := make(matcherHeap) | ||
| extractSelectors(heap, expr) | ||
| replaceMatchers(heap, &expr) | ||
|
|
||
| return expr | ||
| } | ||
|
|
||
| func extractSelectors(selectors matcherHeap, expr parser.Expr) { | ||
| parser.Inspect(expr, func(node parser.Node, nodes []parser.Node) error { | ||
| e, ok := node.(*parser.VectorSelector) | ||
| if !ok { | ||
| return nil | ||
| } | ||
| for _, l := range e.LabelMatchers { | ||
| if l.Name == labels.MetricName { | ||
| selectors.add(l.Name, e.LabelMatchers) | ||
| } | ||
| } | ||
| return nil | ||
| }) | ||
| } | ||
|
|
||
| func replaceMatchers(selectors matcherHeap, expr *parser.Expr) { | ||
| traverse(expr, func(node *parser.Expr) { | ||
| e, ok := (*node).(*parser.VectorSelector) | ||
| if !ok { | ||
| return | ||
| } | ||
|
|
||
| for _, l := range e.LabelMatchers { | ||
| if l.Name == labels.MetricName { | ||
| replacement, found := selectors.findReplacement(l.Name, e.LabelMatchers) | ||
| if found { | ||
| // All replacements are done on metrics only, | ||
| // so we can drop the explicit metric name selector. | ||
| filters := dropMetricName(e.LabelMatchers) | ||
| e.LabelMatchers = replacement | ||
| *node = &FilteredSelector{ | ||
| Filters: filters, | ||
| VectorSelector: e, | ||
| } | ||
| return | ||
| } | ||
| } | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| func dropMetricName(originalMatchers []*labels.Matcher) []*labels.Matcher { | ||
| for i, l := range originalMatchers { | ||
| if l.Name == labels.MetricName { | ||
| originalMatchers = append(originalMatchers[:i], originalMatchers[i+1:]...) | ||
| } | ||
| } | ||
| return originalMatchers | ||
| } | ||
|
|
||
| func matcherToMap(matchers []*labels.Matcher) map[string]*labels.Matcher { | ||
| r := make(map[string]*labels.Matcher, len(matchers)) | ||
| for i := 0; i < len(matchers); i++ { | ||
| r[matchers[i].Name] = matchers[i] | ||
| } | ||
| return r | ||
| } | ||
|
|
||
| // matcherHeap is a set of the most selective label matchers | ||
| // for each metrics discovered in a PromQL expression. | ||
| type matcherHeap map[string][]*labels.Matcher | ||
|
|
||
| func (m matcherHeap) add(metricName string, lessSelective []*labels.Matcher) { | ||
| moreSelective, ok := m[metricName] | ||
| if !ok { | ||
| m[metricName] = lessSelective | ||
| return | ||
| } | ||
|
|
||
| if len(lessSelective) < len(moreSelective) { | ||
| lessSelective, moreSelective = moreSelective, lessSelective | ||
| } | ||
|
|
||
| m[metricName] = moreSelective | ||
| } | ||
|
|
||
| func (m matcherHeap) findReplacement(metricName string, matcher []*labels.Matcher) ([]*labels.Matcher, bool) { | ||
| top, ok := m[metricName] | ||
| if !ok { | ||
| return nil, false | ||
| } | ||
|
|
||
| matcherSet := matcherToMap(matcher) | ||
| topSet := matcherToMap(top) | ||
| for k, v := range topSet { | ||
| m, ok := matcherSet[k] | ||
| if !ok { | ||
| return nil, false | ||
| } | ||
|
|
||
| equals := v.Name == m.Name && v.Type == m.Type && v.Value == m.Value | ||
| if !equals { | ||
| return nil, false | ||
| } | ||
| } | ||
|
|
||
| // The top matcher and input matcher are equal. No replacement needed. | ||
| if len(top) == len(matcherSet) { | ||
| return nil, false | ||
| } | ||
|
|
||
| return top, true | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| package logicalplan | ||
|
|
||
| import "github.com/prometheus/prometheus/promql/parser" | ||
|
|
||
| var DefaultOptimizers = []Optimizer{ | ||
| SortMatchers{}, | ||
| MergeSelectsOptimizer{}, | ||
| } | ||
|
|
||
| type Plan interface { | ||
| RunOptimizers([]Optimizer) parser.Expr | ||
| } | ||
|
|
||
| type Optimizer interface { | ||
| Optimize(parser.Expr) parser.Expr | ||
| } | ||
|
|
||
| type plan struct { | ||
| expr parser.Expr | ||
| } | ||
|
|
||
| func New(expr parser.Expr) Plan { | ||
| return &plan{ | ||
| expr: expr, | ||
| } | ||
| } | ||
|
|
||
| func (p *plan) RunOptimizers(optimizers []Optimizer) parser.Expr { | ||
| for _, o := range optimizers { | ||
| p.expr = o.Optimize(p.expr) | ||
| } | ||
| return p.expr | ||
| } | ||
|
|
||
| func traverse(expr *parser.Expr, transform func(*parser.Expr)) { | ||
| switch node := (*expr).(type) { | ||
| case *parser.VectorSelector: | ||
| transform(expr) | ||
| case *parser.MatrixSelector: | ||
| transform(&node.VectorSelector) | ||
| case *parser.AggregateExpr: | ||
| traverse(&node.Expr, transform) | ||
| case *parser.Call: | ||
| for _, n := range node.Args { | ||
| traverse(&n, transform) | ||
| } | ||
| case *parser.BinaryExpr: | ||
| traverse(&node.LHS, transform) | ||
| traverse(&node.RHS, transform) | ||
| case *parser.UnaryExpr: | ||
| traverse(&node.Expr, transform) | ||
| case *parser.ParenExpr: | ||
| traverse(&node.Expr, transform) | ||
| case *parser.SubqueryExpr: | ||
| traverse(&node.Expr, transform) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| package logicalplan | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/efficientgo/core/testutil" | ||
| "github.com/prometheus/prometheus/promql/parser" | ||
| ) | ||
|
|
||
| func TestDefaultOptimizers(t *testing.T) { | ||
| cases := []struct { | ||
| name string | ||
| expr string | ||
| expected string | ||
| }{ | ||
| { | ||
| name: "common selectors", | ||
| expr: `sum(metric{a="b", c="d"}) / sum(metric{a="b"})`, | ||
| expected: `sum(filter([a="b" c="d"], metric{a="b"})) / sum(metric{a="b"})`, | ||
| }, | ||
| { | ||
| name: "different selectors", | ||
| expr: `sum(metric{a="b"}) / sum(metric{c="d"})`, | ||
| expected: `sum(metric{a="b"}) / sum(metric{c="d"})`, | ||
| }, | ||
| } | ||
|
|
||
| for _, tcase := range cases { | ||
| t.Run(tcase.name, func(t *testing.T) { | ||
| expr, err := parser.ParseExpr(tcase.expr) | ||
| testutil.Ok(t, err) | ||
|
|
||
| plan := New(expr) | ||
| optimizedPlan := plan.RunOptimizers(DefaultOptimizers) | ||
| testutil.Equals(t, tcase.expected, optimizedPlan.String()) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package logicalplan | ||
|
|
||
| import ( | ||
| "sort" | ||
|
|
||
| "github.com/prometheus/prometheus/promql/parser" | ||
| ) | ||
|
|
||
| // SortMatchers sorts all selectors in a selector so that | ||
| // all subsequent optimizers, both in the logical and physical plan | ||
| // can rely on this property. | ||
| type SortMatchers struct{} | ||
|
|
||
| func (m SortMatchers) Optimize(expr parser.Expr) parser.Expr { | ||
| traverse(&expr, func(node *parser.Expr) { | ||
| e, ok := (*node).(*parser.VectorSelector) | ||
| if !ok { | ||
| return | ||
| } | ||
|
|
||
| sort.Slice(e.LabelMatchers, func(i, j int) bool { | ||
| return e.LabelMatchers[i].Name == e.LabelMatchers[j].Name | ||
| }) | ||
|
|
||
| return | ||
| }) | ||
| return expr | ||
| } |
Oops, something went wrong.