NETOBSERV-2187: Implement DSL based on goyacc, replacing keep_entry API

.gitignore

@@ -2,3 +2,4 @@
 /confgenerator
 /bin/
 cover.out
+y.output

Makefile

@@ -197,5 +197,11 @@ else
 	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${IMAGE} docker://${IMAGE};
 endif
 
+.PHONY: goyacc
+goyacc: ## Regenerate filters query langage
+	@echo "### Regenerate filters query langage"
+	GOFLAGS="" go install golang.org/x/tools/cmd/[email protected]
+	goyacc -o pkg/dsl/expr.y.go pkg/dsl/expr.y
+
 include .mk/development.mk
 include .mk/shortcuts.mk

README.md

@@ -386,6 +386,16 @@ removal of only the `SrcPort` key and value
 Using `remove_entry_if_equal` will remove the entry if the specified field exists and is equal to the specified value.
 Using `remove_entry_if_not_equal` will remove the entry if the specified field exists and is not equal to the specified value.
 
+#### Transform Filter: query language
+
+Alternatively, a query language allows to filter flows, keeping entries rather than removing them.
+
+```
+(srcnamespace="netobserv" OR (srcnamespace="ingress" AND dstnamespace="netobserv")) AND srckind!="service"
+```
+
+[See here](./docs/filtering.md) for more information about this language.
+
 ### Transform Network
 
 `transform network` provides specific functionality that is useful for transformation of network flow-logs:
@@ -945,6 +955,7 @@ Images
   image-push            Push MULTIARCH_TARGETS images  
   manifest-build        Build MULTIARCH_TARGETS manifest  
   manifest-push         Push MULTIARCH_TARGETS manifest  
+  goyacc                Regenerate filters query langage  
 
 kubernetes  
   deploy                Deploy the image  

docs/api.md

@@ -162,7 +162,7 @@ Following is the supported API format for filter transformations:
                     remove_entry_if_equal: removes the entry if the field value equals specified value
                     remove_entry_if_not_equal: removes the entry if the field value does not equal specified value
                     remove_entry_all_satisfied: removes the entry if all of the defined rules are satisfied
-                    keep_entry_all_satisfied: keeps the entry if the set of rules are all satisfied
+                    keep_entry_query: keeps the entry if it matches the query
                     add_field: adds (input) field to the entry; overrides previous value if present (key=input, value=value)
                     add_field_if_doesnt_exist: adds a field to the entry if the field does not exist
                     add_field_if: add output field set to assignee if input field satisfies criteria from parameters field
@@ -188,18 +188,7 @@ Following is the supported API format for filter transformations:
                              input: entry input field
                              value: specified value of input field:
                              castInt: set true to cast the value field as an int (numeric values are float64 otherwise)
-                 keepEntryAllSatisfied: configuration for keep_entry rule
-                         type: (enum) one of the following:
-                            keep_entry_if_exists: keeps the entry if the field exists
-                            keep_entry_if_doesnt_exist: keeps the entry if the field does not exist
-                            keep_entry_if_equal: keeps the entry if the field value equals specified value
-                            keep_entry_if_not_equal: keeps the entry if the field value does not equal specified value
-                            keep_entry_if_regex_match: keeps the entry if the field value matches the specified regex
-                            keep_entry_if_not_regex_match: keeps the entry if the field value does not match the specified regex
-                         keepEntry: configuration for keep_entry_* rules
-                             input: entry input field
-                             value: specified value of input field:
-                             castInt: set true to cast the value field as an int (numeric values are float64 otherwise)
+                 keepEntryQuery: configuration for keep_entry rule
                  keepEntrySampling: sampling value for keep_entry type: 1 flow on <sampling> is kept
                  addField: configuration for add_field rule
                      input: entry input field

docs/filtering.md

@@ -0,0 +1,68 @@
+# FLP filtering language
+
+Flowlogs-pipeline uses a simple query language to filter network flows:
+
+```
+(srcnamespace="netobserv" OR (srcnamespace="ingress" AND dstnamespace="netobserv")) AND srckind!="service"
+```
+
+The syntax includes:
+
+- Logical boolean operators (case insensitive)
+  - `and`
+  - `or`
+- Comparison operators
+  - equals `=`
+  - not equals `!=`
+  - matches regexp `=~`
+  - not matches regexp `!~`
+  - greater than (or equal) `>` / `>=`
+  - less than (or equal) `<` / `<=`
+- Unary operations
+  - field is present: `with(field)`
+  - field is absent: `without(field)`
+- Parenthesis-based priority
+
+## API integration
+
+The language is currently integrated in the "keep_entry" transform/filtering API. Example:
+
+```yaml
+    transform:
+      type: filter
+      filter:
+        rules:
+        - type: keep_entry_query
+          keepEntryQuery: (namespace="A" and with(workload)) or service=~"abc.+"
+          keepEntrySampling: 10 # Optionally, a sampling ratio can be associated with the filter
+```
+
+## Integration with the NetObserv operator
+
+In the [NetObserv operator](https://github.com/netobserv/network-observability-operator), the filtering query language is used in `FlowCollector` `spec.processor.filters`. Example:
+
+```yaml
+spec:
+  processor:
+    filters:
+      - query: |
+          (SrcK8S_Namespace="netobserv" OR (SrcK8S_Namespace="openshift-ingress" AND DstK8S_Namespace="netobserv"))
+        outputTarget: Loki  # The filter can target a specific output (such as Loki logs or exported data), or all outputs.
+        sampling: 10        # Optionally, a sampling ratio can be associated with the filter
+```
+
+See also the [list of field names](https://github.com/netobserv/network-observability-operator/blob/main/docs/flows-format.adoc) that are available for queries, and the [API documentation](https://github.com/netobserv/network-observability-operator/blob/main/docs/FlowCollector.md#flowcollectorspecprocessorfiltersindex-1).
+
+## Internals
+
+This language is designed using [Yacc](https://en.wikipedia.org/wiki/Yacc) / goyacc.
+
+The [definition file](../pkg/dsl/expr.y) describes the syntax based on a list of tokens. It is derived to a [go source file](../pkg/dsl/expr.y.go) using [goyacc](https://pkg.go.dev/golang.org/x/tools/cmd/goyacc), which defines constants for the tokens, among other things. The [lexer](../pkg/dsl/lexer.go) file defines structures and helpers that can be used from `expr.y`, the logic used to interpret the language in a structured way, and is also where actual characters/strings are mapped to syntax tokens. Finally, [eval.go](../pkg/dsl/eval.go) runs the desired query on actual data.
+
+When adding features to the language, you'll likely have to change `expr.y` and `lexer.go`.
+
+To regenerate `expr.y.go`, run:
+
+```bash
+make goyacc
+```

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/prometheus/client_golang v1.22.0
 	github.com/prometheus/client_model v0.6.2
 	github.com/prometheus/common v0.63.0
+	github.com/prometheus/prometheus v1.8.2-0.20201028100903-3245b3267b24
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.9.1
@@ -114,7 +115,6 @@ require (
 	github.com/pion/transport/v2 v2.2.10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/procfs v0.16.0 // indirect
-	github.com/prometheus/prometheus v1.8.2-0.20201028100903-3245b3267b24 // indirect
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/safchain/ethtool v0.5.10 // indirect

pkg/api/transform_filter.go

@@ -37,7 +37,7 @@ const (
 	RemoveEntryIfEqual       TransformFilterEnum = "remove_entry_if_equal"        // removes the entry if the field value equals specified value
 	RemoveEntryIfNotEqual    TransformFilterEnum = "remove_entry_if_not_equal"    // removes the entry if the field value does not equal specified value
 	RemoveEntryAllSatisfied  TransformFilterEnum = "remove_entry_all_satisfied"   // removes the entry if all of the defined rules are satisfied
-	KeepEntryAllSatisfied    TransformFilterEnum = "keep_entry_all_satisfied"     // keeps the entry if the set of rules are all satisfied
+	KeepEntryQuery           TransformFilterEnum = "keep_entry_query"             // keeps the entry if it matches the query
 	AddField                 TransformFilterEnum = "add_field"                    // adds (input) field to the entry; overrides previous value if present (key=input, value=value)
 	AddFieldIfDoesntExist    TransformFilterEnum = "add_field_if_doesnt_exist"    // adds a field to the entry if the field does not exist
 	AddFieldIf               TransformFilterEnum = "add_field_if"                 // add output field set to assignee if input field satisfies criteria from parameters field
@@ -56,23 +56,12 @@ const (
 	RemoveEntryIfNotEqualD    TransformFilterRemoveEntryEnum = "remove_entry_if_not_equal"    // removes the entry if the field value does not equal specified value
 )
 
-type TransformFilterKeepEntryEnum string
-
-const (
-	KeepEntryIfExists        TransformFilterKeepEntryEnum = "keep_entry_if_exists"          // keeps the entry if the field exists
-	KeepEntryIfDoesntExist   TransformFilterKeepEntryEnum = "keep_entry_if_doesnt_exist"    // keeps the entry if the field does not exist
-	KeepEntryIfEqual         TransformFilterKeepEntryEnum = "keep_entry_if_equal"           // keeps the entry if the field value equals specified value
-	KeepEntryIfNotEqual      TransformFilterKeepEntryEnum = "keep_entry_if_not_equal"       // keeps the entry if the field value does not equal specified value
-	KeepEntryIfRegexMatch    TransformFilterKeepEntryEnum = "keep_entry_if_regex_match"     // keeps the entry if the field value matches the specified regex
-	KeepEntryIfNotRegexMatch TransformFilterKeepEntryEnum = "keep_entry_if_not_regex_match" // keeps the entry if the field value does not match the specified regex
-)
-
 type TransformFilterRule struct {
 	Type                    TransformFilterEnum              `yaml:"type,omitempty" json:"type,omitempty" doc:"(enum) one of the following:"`
 	RemoveField             *TransformFilterGenericRule      `yaml:"removeField,omitempty" json:"removeField,omitempty" doc:"configuration for remove_field rule"`
 	RemoveEntry             *TransformFilterGenericRule      `yaml:"removeEntry,omitempty" json:"removeEntry,omitempty" doc:"configuration for remove_entry_* rules"`
 	RemoveEntryAllSatisfied []*RemoveEntryRule               `yaml:"removeEntryAllSatisfied,omitempty" json:"removeEntryAllSatisfied,omitempty" doc:"configuration for remove_entry_all_satisfied rule"`
-	KeepEntryAllSatisfied   []*KeepEntryRule                 `yaml:"keepEntryAllSatisfied,omitempty" json:"keepEntryAllSatisfied,omitempty" doc:"configuration for keep_entry rule"`
+	KeepEntryQuery          string                           `yaml:"keepEntryQuery,omitempty" json:"keepEntryQuery,omitempty" doc:"configuration for keep_entry rule"`
 	KeepEntrySampling       uint16                           `yaml:"keepEntrySampling,omitempty" json:"keepEntrySampling,omitempty" doc:"sampling value for keep_entry type: 1 flow on <sampling> is kept"`
 	AddField                *TransformFilterGenericRule      `yaml:"addField,omitempty" json:"addField,omitempty" doc:"configuration for add_field rule"`
 	AddFieldIfDoesntExist   *TransformFilterGenericRule      `yaml:"addFieldIfDoesntExist,omitempty" json:"addFieldIfDoesntExist,omitempty" doc:"configuration for add_field_if_doesnt_exist rule"`
@@ -93,9 +82,6 @@ func (r *TransformFilterRule) preprocess() {
 	for i := range r.RemoveEntryAllSatisfied {
 		r.RemoveEntryAllSatisfied[i].RemoveEntry.preprocess()
 	}
-	for i := range r.KeepEntryAllSatisfied {
-		r.KeepEntryAllSatisfied[i].KeepEntry.preprocess()
-	}
 	for i := range r.ConditionalSampling {
 		r.ConditionalSampling[i].preprocess()
 	}
@@ -127,11 +113,6 @@ type RemoveEntryRule struct {
 	RemoveEntry *TransformFilterGenericRule    `yaml:"removeEntry,omitempty" json:"removeEntry,omitempty" doc:"configuration for remove_entry_* rules"`
 }
 
-type KeepEntryRule struct {
-	Type      TransformFilterKeepEntryEnum `yaml:"type,omitempty" json:"type,omitempty" doc:"(enum) one of the following:"`
-	KeepEntry *TransformFilterGenericRule  `yaml:"keepEntry,omitempty" json:"keepEntry,omitempty" doc:"configuration for keep_entry_* rules"`
-}
-
 type SamplingCondition struct {
 	Value uint16             `yaml:"value,omitempty" json:"value,omitempty" doc:"sampling value: 1 flow on <sampling> is kept"`
 	Rules []*RemoveEntryRule `yaml:"rules,omitempty" json:"rules,omitempty" doc:"rules to be satisfied for this sampling configuration"`

pkg/dsl/eval.go

@@ -0,0 +1,33 @@
+package dsl
+
+import (
+	"github.com/netobserv/flowlogs-pipeline/pkg/config"
+	"github.com/netobserv/flowlogs-pipeline/pkg/utils/filters"
+)
+
+type tree struct {
+	logicalOp string
+	children  []*tree
+	predicate filters.Predicate
+}
+
+func (t *tree) apply(flow config.GenericMap) bool {
+	if t.predicate != nil {
+		return t.predicate(flow)
+	}
+	if t.logicalOp == operatorAnd {
+		for _, child := range t.children {
+			if !child.apply(flow) {
+				return false
+			}
+		}
+		return true
+	}
+	// t.logicalOp == operatorOr
+	for _, child := range t.children {
+		if child.apply(flow) {
+			return true
+		}
+	}
+	return false
+}