chore(pe): Replace standard library PE parser with saferwall parser (#…

…161)

* use saferwall/pe package for PE introspection

* avoid returning lang id, introduce just string sanity check. Expand test cases and other minor adjustments

* introduce VA (Virtual Address) region area reading and `ParseMem` function for fetching remote PE

* introduce ParseMem tests and small adjustments

* VA region tests, upgrade saferwall/pe and set up options for directory parsing

* document structures/functions

* update the stats structure with new expvar metrics

* address lint warnings and adjust tests

* fix lint warnings in tests

cmd/fibratus/app/stats.go

@@ -96,8 +96,9 @@ type Stats struct {
 	OutputAMQPPublishErrors             int            `json:"output.amqp.publish.errors"`
 	OutputConsoleErrors                 int            `json:"output.console.errors"`
 	OutputNullBlackholeEvents           int            `json:"output.null.blackhole.events"`
-	PeFailedResourceEntryReads          int            `json:"pe.failed.resource.entry.reads"`
-	PeMaxResourceEntriesExceeded        int            `json:"pe.max.resource.entries.exceeded"`
+	PeSkippedImages                     int            `json:"pe.skipped.images"`
+	PeDirectoryParseErrors              int            `json:"pe.directory.parse.errors"`
+	PeVersionResourcesParseErrors       int            `json:"pe.version.resources.parse.errors"`
 	ProcessCount                        int            `json:"process.count"`
 	ProcessModuleCount                  int            `json:"process.module.count"`
 	ProcessLookupFailureCount           map[int]int    `json:"process.lookup.failure.count"`

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/qmuntal/stateless v1.6.0
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
+	github.com/saferwall/pe v1.4.0
 	github.com/sirupsen/logrus v1.4.1
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.5
@@ -39,6 +40,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/antchfx/xpath v1.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/edsrzf/mmap-go v1.1.0 // indirect
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.7 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
@@ -65,6 +67,7 @@ require (
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
 	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/net v0.7.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect

go.sum

@@ -36,6 +36,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
+github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
@@ -153,6 +155,8 @@ github.com/qmuntal/stateless v1.6.0/go.mod h1:cWTwXu9ey+FxI0fHvDi1nGCtpYa8N1X2aO
 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/saferwall/pe v1.4.0 h1:5aZVTAEqYYwbC078GDPtml6pVcY6iLbTgO86leqp6K8=
+github.com/saferwall/pe v1.4.0/go.mod h1:Fh6INnfm6vUTCW1w/pZ59aWQhlOwqBVxVACtadb3UbA=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -212,6 +216,8 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:
 github.com/yuin/goldmark v1.5.2 h1:ALmeCk/px5FSm1MAcFBAsVKZjDuMVj8Tm7FFIlMJnqU=
 github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak=
+go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -254,6 +260,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/filter/ql/functions/entropy.go

@@ -20,11 +20,15 @@ package functions
 
 import (
 	"fmt"
-	"math"
+	"github.com/rabbitstack/fibratus/pkg/util/entropy"
+	"strings"
 )
 
 const (
-	shannonAlgo = "shannon"
+	// Shannon computes the string entropy by employing
+	// the Shannon algorithm.
+	// https://en.wikipedia.org/wiki/Entropy_(information_theory)
+	Shannon = "shannon"
 )
 
 // Entropy measures the string entropy
@@ -36,12 +40,12 @@ func (f Entropy) Call(args []interface{}) (interface{}, bool) {
 	}
 	s := parseString(0, args)
 	if len(args) == 1 {
-		return shannon(s), true
+		return entropy.Shannon(s), true
 	}
 	algo := parseString(1, args)
 	switch algo {
-	case shannonAlgo:
-		return shannon(s), true
+	case Shannon:
+		return entropy.Shannon(s), true
 	default:
 		return false, false
 	}
@@ -58,8 +62,9 @@ func (f Entropy) Desc() FunctionDesc {
 			if len(args) == 1 {
 				return nil
 			}
-			if len(args) > 1 && args[1] != shannonAlgo {
-				return fmt.Errorf("unsupported entropy algorithm: %s. Availiable algorithms: shannon", args[1])
+			if len(args) > 1 && args[1] != Shannon {
+				return fmt.Errorf("unsupported entropy algorithm: %s. Availiable algorithms: %s", args[1],
+					strings.Join([]string{Shannon}, "|"))
 			}
 			return nil
 		},
@@ -68,22 +73,3 @@ func (f Entropy) Desc() FunctionDesc {
 }
 
 func (f Entropy) Name() Fn { return EntropyFn }
-
-// shannon measures the Shannon entropy of a string.
-func shannon(value string) int {
-	frq := make(map[rune]float64)
-
-	//get frequency of characters
-	for _, i := range value {
-		frq[i]++
-	}
-
-	var sum float64
-
-	for _, v := range frq {
-		f := v / float64(len(value))
-		sum += f * math.Log2(f)
-	}
-
-	return int(math.Ceil(sum*-1)) * len(value)
-}

pkg/pe/config.go

@@ -24,8 +24,6 @@ package pe
 import (
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
-	"path/filepath"
-	"strings"
 )
 
 const (
@@ -62,13 +60,3 @@ func AddFlags(flags *pflag.FlagSet) {
 	flags.Bool(readSections, false, "Indicates if full section inspection is allowed")
 	flags.StringSlice(excludedImages, []string{}, "Contains a list of comma-separated images names that are excluded from PE parsing")
 }
-
-// ShouldSkipProcess determines whether the specified filename name is ignored by PE reader.
-func (c Config) shouldSkipImage(filename string) bool {
-	for _, img := range c.ExcludedImages {
-		if strings.EqualFold(img, filepath.Base(filename)) {
-			return true
-		}
-	}
-	return false
-}

pkg/pe/doc.go

@@ -17,5 +17,5 @@
  */
 
 // Package pe contains different facilities for dealing with Portable Executable specifics and digging out valuable insights
-// from it.
+// from PE.
 package pe