From 44f127833d42f487fe9cfece40b0e49bfdfbf978 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Thu, 23 Sep 2021 16:48:58 +0800 Subject: [PATCH 01/34] add chimera library --- chimera/common.go | 7 ++++ chimera/common_test.go | 14 ++++++++ chimera/internal.go | 77 ++++++++++++++++++++++++++++++++++++++++ hyperscan/internal.go | 27 +++++++------- hyperscan/internal_v5.go | 7 ++++ 5 files changed, 120 insertions(+), 12 deletions(-) create mode 100644 chimera/common.go create mode 100644 chimera/common_test.go create mode 100644 chimera/internal.go diff --git a/chimera/common.go b/chimera/common.go new file mode 100644 index 0000000..396b75a --- /dev/null +++ b/chimera/common.go @@ -0,0 +1,7 @@ +package chimera + +// Version identify this release version. +// +// The return version is a string containing the version number of this release +// build and the date of the build. +func Version() string { return chVersion() } diff --git a/chimera/common_test.go b/chimera/common_test.go new file mode 100644 index 0000000..1d0934c --- /dev/null +++ b/chimera/common_test.go @@ -0,0 +1,14 @@ +package chimera_test + +import ( + "testing" + + "github.com/flier/gohs/chimera" + . "github.com/smartystreets/goconvey/convey" +) + +func TestChimera(t *testing.T) { + Convey("Given a chimera runtimes", t, func() { + So(chimera.Version(), ShouldNotBeEmpty) + }) +} diff --git a/chimera/internal.go b/chimera/internal.go new file mode 100644 index 0000000..53fd22b --- /dev/null +++ b/chimera/internal.go @@ -0,0 +1,77 @@ +package chimera + +/* +#cgo pkg-config: libch libhs +#cgo linux LDFLAGS: -lm -lpcre +#cgo darwin LDFLAGS: -lpcre + +#include +#include +#include + +#include +*/ +import "C" +import "fmt" + +// ChError represents an error +type ChError C.ch_error_t + +const ( + // ErrSuccess is the error returned if the engine completed normally. + ErrSuccess ChError = C.CH_SUCCESS + // ErrInvalid is the error returned if a parameter passed to this function was invalid. + ErrInvalid ChError = C.CH_INVALID + // ErrNoMemory is the error returned if a memory allocation failed. + ErrNoMemory ChError = C.CH_NOMEM + // ErrScanTerminated is the error returned if the engine was terminated by callback. + ErrScanTerminated ChError = C.CH_SCAN_TERMINATED + // ErrCompileError is the error returned if the pattern compiler failed. + ErrCompileError ChError = C.CH_COMPILER_ERROR + // ErrDatabaseVersionError is the error returned if the given database was built for a different version of Hyperscan. + ErrDatabaseVersionError ChError = C.CH_DB_VERSION_ERROR + // ErrDatabasePlatformError is the error returned if the given database was built for a different platform (i.e., CPU type). + ErrDatabasePlatformError ChError = C.CH_DB_PLATFORM_ERROR + // ErrDatabaseModeError is the error returned if the given database was built for a different mode of operation. + ErrDatabaseModeError ChError = C.CH_DB_MODE_ERROR + // ErrBadAlign is the error returned if a parameter passed to this function was not correctly aligned. + ErrBadAlign ChError = C.CH_BAD_ALIGN + // ErrBadAlloc is the error returned if the memory allocator did not correctly return memory suitably aligned. + ErrBadAlloc ChError = C.CH_BAD_ALLOC + // ErrScratchInUse is the error returned if the scratch region was already in use. + ErrScratchInUse ChError = C.CH_SCRATCH_IN_USE + // ErrUnknown is the unexpected internal error from Hyperscan. + ErrUnknownHSError ChError = C.CH_UNKNOWN_HS_ERROR +) + +var chErrorMessages = map[ChError]string{ + C.CH_SUCCESS: "The engine completed normally.", + C.CH_INVALID: "A parameter passed to this function was invalid.", + C.CH_NOMEM: "A memory allocation failed.", + C.CH_SCAN_TERMINATED: "The engine was terminated by callback.", + C.CH_COMPILER_ERROR: "The pattern compiler failed.", + C.CH_DB_VERSION_ERROR: "The given database was built for a different version of Hyperscan.", + C.CH_DB_PLATFORM_ERROR: "The given database was built for a different platform (i.e., CPU type).", + C.CH_DB_MODE_ERROR: "The given database was built for a different mode of operation.", + C.CH_BAD_ALIGN: "A parameter passed to this function was not correctly aligned.", + C.CH_BAD_ALLOC: "The memory allocator did not correctly return aligned memory.", + C.CH_SCRATCH_IN_USE: "The scratch region was already in use.", + C.CH_UNKNOWN_HS_ERROR: "Unexpected internal error from Hyperscan.", +} + +func (e ChError) Error() string { + if msg, exists := chErrorMessages[e]; exists { + return msg + } + + return fmt.Sprintf("unexpected error, %d", int(e)) +} + +type ( + chDatabase *C.ch_database_t + chScratch *C.ch_scratch_t +) + +func chVersion() string { + return C.GoString(C.ch_version()) +} diff --git a/hyperscan/internal.go b/hyperscan/internal.go index 0babab6..106e4ca 100644 --- a/hyperscan/internal.go +++ b/hyperscan/internal.go @@ -269,21 +269,24 @@ const ( ErrScratchInUse HsError = C.HS_SCRATCH_IN_USE // ErrArchError is the error returned if unsupported CPU architecture. ErrArchError HsError = C.HS_ARCH_ERROR + // ErrInsufficientSpace is the error returned if provided buffer was too small. + ErrInsufficientSpace HsError = C.HS_INSUFFICIENT_SPACE ) var hsErrorMessages = map[HsError]string{ - C.HS_SUCCESS: "The engine completed normally.", - C.HS_INVALID: "A parameter passed to this function was invalid.", - C.HS_NOMEM: "A memory allocation failed.", - C.HS_SCAN_TERMINATED: "The engine was terminated by callback.", - C.HS_COMPILER_ERROR: "The pattern compiler failed.", - C.HS_DB_VERSION_ERROR: "The given database was built for a different version of Hyperscan.", - C.HS_DB_PLATFORM_ERROR: "The given database was built for a different platform (i.e., CPU type).", - C.HS_DB_MODE_ERROR: "The given database was built for a different mode of operation.", - C.HS_BAD_ALIGN: "A parameter passed to this function was not correctly aligned.", - C.HS_BAD_ALLOC: "The memory allocator did not correctly return aligned memory.", - C.HS_SCRATCH_IN_USE: "The scratch region was already in use.", - C.HS_ARCH_ERROR: "Unsupported CPU architecture.", + C.HS_SUCCESS: "The engine completed normally.", + C.HS_INVALID: "A parameter passed to this function was invalid.", + C.HS_NOMEM: "A memory allocation failed.", + C.HS_SCAN_TERMINATED: "The engine was terminated by callback.", + C.HS_COMPILER_ERROR: "The pattern compiler failed.", + C.HS_DB_VERSION_ERROR: "The given database was built for a different version of Hyperscan.", + C.HS_DB_PLATFORM_ERROR: "The given database was built for a different platform (i.e., CPU type).", + C.HS_DB_MODE_ERROR: "The given database was built for a different mode of operation.", + C.HS_BAD_ALIGN: "A parameter passed to this function was not correctly aligned.", + C.HS_BAD_ALLOC: "The memory allocator did not correctly return aligned memory.", + C.HS_SCRATCH_IN_USE: "The scratch region was already in use.", + C.HS_ARCH_ERROR: "Unsupported CPU architecture.", + C.HS_INSUFFICIENT_SPACE: "Provided buffer was too small.", } func (e HsError) Error() string { diff --git a/hyperscan/internal_v5.go b/hyperscan/internal_v5.go index e57f8d8..7b2ac2f 100644 --- a/hyperscan/internal_v5.go +++ b/hyperscan/internal_v5.go @@ -13,6 +13,11 @@ import ( "unsafe" ) +const ( + // ErrUnknown is an unexpected internal error. + ErrUnknown HsError = C.HS_UNKNOWN_ERROR +) + const ( // Combination represents logical combination. Combination CompileFlag = C.HS_FLAG_COMBINATION @@ -21,6 +26,8 @@ const ( ) func init() { + hsErrorMessages[C.HS_UNKNOWN_ERROR] = "Unexpected internal error." + compileFlags['C'] = Combination compileFlags['Q'] = Quiet } From fad9e2587a43de3ff4b6b11789f4b47f967b7f7f Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Thu, 23 Sep 2021 17:08:37 +0800 Subject: [PATCH 02/34] force to use C++ compiler --- chimera/dummy.cxx | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 chimera/dummy.cxx diff --git a/chimera/dummy.cxx b/chimera/dummy.cxx new file mode 100644 index 0000000..e69de29 From 95450aafaa101d3c59a536af8b555cd3e390f55b Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sat, 25 Sep 2021 00:41:21 +0800 Subject: [PATCH 03/34] refactor internal and add chimera --- chimera/common.go | 71 +- chimera/common_test.go | 3 +- chimera/compile.go | 97 ++ chimera/doc.go | 56 + chimera/pattern.go | 96 ++ chimera/pattern_test.go | 82 ++ chimera/runtime.go | 6 + examples/simplegrep/main.go | 2 +- hyperscan/api.go | 47 +- hyperscan/block.go | 194 +++ hyperscan/common.go | 88 +- hyperscan/compile.go | 294 ++-- hyperscan/compile_test.go | 126 +- hyperscan/compile_v5.go | 185 +-- hyperscan/compile_v5_test.go | 86 -- hyperscan/doc.go | 45 +- hyperscan/example_api_test.go | 4 +- hyperscan/internal.go | 1253 ----------------- hyperscan/internal_test.go | 674 --------- hyperscan/internal_v54.go | 18 - hyperscan/literal.go | 214 +++ hyperscan/literal_test.go | 98 ++ hyperscan/pattern.go | 381 +++++ hyperscan/pattern_test.go | 117 ++ hyperscan/platform.go | 56 + hyperscan/platform_test.go | 21 + hyperscan/platform_v54.go | 22 + hyperscan/runtime.go | 646 +-------- hyperscan/scratch.go | 60 + hyperscan/stream.go | 368 +++++ hyperscan/vectored.go | 43 + internal/ch/allocator.go | 208 +++ internal/ch/common.go | 45 + internal/ch/common_test.go | 21 + internal/ch/compile.go | 186 +++ {chimera => internal/ch}/dummy.cxx | 0 chimera/internal.go => internal/ch/error.go | 60 +- internal/ch/link.go | 8 + internal/ch/runtime.go | 160 +++ {hyperscan => internal}/handle/go1_15.go | 0 {hyperscan => internal}/handle/go1_17.go | 0 {hyperscan => internal}/handle/go1_9.go | 0 {hyperscan => internal}/handle/handle.go | 0 internal/hs/allocator.go | 248 ++++ internal/hs/allocator_test.go | 138 ++ internal/hs/common.go | 135 ++ internal/hs/common_test.go | 126 ++ internal/hs/compile.go | 434 ++++++ internal/hs/compile_test.go | 135 ++ .../hs/compile_v5.go | 45 +- internal/hs/compile_v54.go | 22 + {hyperscan => internal/hs}/dummy.cxx | 0 internal/hs/error.go | 61 + internal/hs/error_v5.go | 18 + internal/hs/link.go | 7 + internal/hs/runtime.go | 154 ++ internal/hs/runtime_test.go | 112 ++ internal/hs/scratch.go | 52 + internal/hs/scratch_test.go | 76 + internal/hs/stream.go | 174 +++ internal/hs/stream_test.go | 96 ++ 61 files changed, 4871 insertions(+), 3303 deletions(-) create mode 100644 chimera/compile.go create mode 100644 chimera/doc.go create mode 100644 chimera/pattern.go create mode 100644 chimera/pattern_test.go create mode 100644 chimera/runtime.go create mode 100644 hyperscan/block.go delete mode 100644 hyperscan/internal.go delete mode 100644 hyperscan/internal_test.go delete mode 100644 hyperscan/internal_v54.go create mode 100644 hyperscan/literal.go create mode 100644 hyperscan/literal_test.go create mode 100644 hyperscan/pattern.go create mode 100644 hyperscan/pattern_test.go create mode 100644 hyperscan/platform.go create mode 100644 hyperscan/platform_test.go create mode 100644 hyperscan/platform_v54.go create mode 100644 hyperscan/scratch.go create mode 100644 hyperscan/stream.go create mode 100644 hyperscan/vectored.go create mode 100644 internal/ch/allocator.go create mode 100644 internal/ch/common.go create mode 100644 internal/ch/common_test.go create mode 100644 internal/ch/compile.go rename {chimera => internal/ch}/dummy.cxx (100%) rename chimera/internal.go => internal/ch/error.go (65%) create mode 100644 internal/ch/link.go create mode 100644 internal/ch/runtime.go rename {hyperscan => internal}/handle/go1_15.go (100%) rename {hyperscan => internal}/handle/go1_17.go (100%) rename {hyperscan => internal}/handle/go1_9.go (100%) rename {hyperscan => internal}/handle/handle.go (100%) create mode 100644 internal/hs/allocator.go create mode 100644 internal/hs/allocator_test.go create mode 100644 internal/hs/common.go create mode 100644 internal/hs/common_test.go create mode 100644 internal/hs/compile.go create mode 100644 internal/hs/compile_test.go rename hyperscan/internal_v5.go => internal/hs/compile_v5.go (65%) create mode 100644 internal/hs/compile_v54.go rename {hyperscan => internal/hs}/dummy.cxx (100%) create mode 100644 internal/hs/error.go create mode 100644 internal/hs/error_v5.go create mode 100644 internal/hs/link.go create mode 100644 internal/hs/runtime.go create mode 100644 internal/hs/runtime_test.go create mode 100644 internal/hs/scratch.go create mode 100644 internal/hs/scratch_test.go create mode 100644 internal/hs/stream.go create mode 100644 internal/hs/stream_test.go diff --git a/chimera/common.go b/chimera/common.go index 396b75a..7ee1126 100644 --- a/chimera/common.go +++ b/chimera/common.go @@ -1,7 +1,76 @@ package chimera +import ( + "github.com/flier/gohs/internal/ch" +) + +// Error is the type for errors returned by Chimera functions. +type Error = ch.Error + +const ( + // ErrSuccess is the error returned if the engine completed normally. + ErrSuccess Error = ch.ErrSuccess + // ErrInvalid is the error returned if a parameter passed to this function was invalid. + ErrInvalid Error = ch.ErrInvalid + // ErrNoMemory is the error returned if a memory allocation failed. + ErrNoMemory Error = ch.ErrNoMemory + // ErrScanTerminated is the error returned if the engine was terminated by callback. + ErrScanTerminated Error = ch.ErrScanTerminated + // ErrCompileError is the error returned if the pattern compiler failed. + ErrCompileError Error = ch.ErrCompileError + // ErrDatabaseVersionError is the error returned if the given database was built + // for a different version of the Chimera matcher. + ErrDatabaseVersionError Error = ch.ErrDatabaseVersionError + // ErrDatabasePlatformError is the error returned if the given database was built for a different platform. + ErrDatabasePlatformError Error = ch.ErrDatabasePlatformError + // ErrDatabaseModeError is the error returned if the given database was built for a different mode of operation. + ErrDatabaseModeError Error = ch.ErrDatabaseModeError + // ErrBadAlign is the error returned if a parameter passed to this function was not correctly aligned. + ErrBadAlign Error = ch.ErrBadAlign + // ErrBadAlloc is the error returned if the memory allocator did not correctly return memory suitably aligned. + ErrBadAlloc Error = ch.ErrBadAlloc + // ErrScratchInUse is the error returned if the scratch region was already in use. + ErrScratchInUse Error = ch.ErrScratchInUse + // ErrUnknown is the unexpected internal error from Hyperscan. + ErrUnknownHSError Error = ch.ErrUnknownHSError +) + +// DbInfo identify the version and platform information for the supplied database. +type DbInfo string // nolint: stylecheck + +func (i DbInfo) String() string { return string(i) } + +// Database is an immutable database that can be used by the Hyperscan scanning API. +type Database interface { + // Provides information about a database. + Info() (DbInfo, error) + + // Provides the size of the given database in bytes. + Size() (int, error) + + // Free a compiled pattern database. + Close() error +} + +type database struct { + db ch.Database +} + +func (d *database) Size() (int, error) { return ch.DatabaseSize(d.db) } // nolint: wrapcheck + +func (d *database) Info() (DbInfo, error) { + i, err := ch.DatabaseInfo(d.db) + if err != nil { + return "", err //nolint: wrapcheck + } + + return DbInfo(i), nil +} + +func (d *database) Close() error { return ch.FreeDatabase(d.db) } // nolint: wrapcheck + // Version identify this release version. // // The return version is a string containing the version number of this release // build and the date of the build. -func Version() string { return chVersion() } +func Version() string { return ch.Version() } diff --git a/chimera/common_test.go b/chimera/common_test.go index 1d0934c..6e25829 100644 --- a/chimera/common_test.go +++ b/chimera/common_test.go @@ -3,8 +3,9 @@ package chimera_test import ( "testing" - "github.com/flier/gohs/chimera" . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/chimera" ) func TestChimera(t *testing.T) { diff --git a/chimera/compile.go b/chimera/compile.go new file mode 100644 index 0000000..c044ffd --- /dev/null +++ b/chimera/compile.go @@ -0,0 +1,97 @@ +package chimera + +import ( + "fmt" + "strconv" + + "github.com/flier/gohs/internal/ch" +) + +// A type containing error details that is returned by the compile calls on failure. +// +// The caller may inspect the values returned in this type to determine the cause of failure. +type CompileError = ch.CompileError + +// CompileFlag represents a pattern flag. +type CompileFlag = ch.CompileFlag + +const ( + // Caseless represents set case-insensitive matching. + Caseless CompileFlag = ch.Caseless + // DotAll represents matching a `.` will not exclude newlines. + DotAll CompileFlag = ch.DotAll + // MultiLine set multi-line anchoring. + MultiLine CompileFlag = ch.MultiLine + // SingleMatch set single-match only mode. + SingleMatch CompileFlag = ch.SingleMatch + // Utf8Mode enable UTF-8 mode for this expression. + Utf8Mode CompileFlag = ch.Utf8Mode + // UnicodeProperty enable Unicode property support for this expression. + UnicodeProperty CompileFlag = ch.UnicodeProperty +) + +/* +ParseCompileFlag parse the compile pattern flags from string + + i Caseless Case-insensitive matching + s DotAll Dot (.) will match newlines + m MultiLine Multi-line anchoring + H SingleMatch Report match ID at most once (`o` deprecated) + 8 Utf8Mode UTF-8 mode (`u` deprecated) + W UnicodeProperty Unicode property support (`p` deprecated) +*/ +func ParseCompileFlag(s string) (CompileFlag, error) { + var flags CompileFlag + + for _, c := range s { + if flag, exists := ch.CompileFlags[c]; exists { + flags |= flag + } else { + return 0, fmt.Errorf("flag `%c`, %w", c, ErrInvalid) + } + } + + return flags, nil +} + +// CompileMode flags. +type CompileMode = ch.CompileMode + +const ( + // Disable capturing groups. + NoGroups CompileMode = ch.NoGroups + + // Enable capturing groups. + Groups CompileMode = ch.Groups +) + +// Compile a regular expression and returns, if successful, +// a pattern database in the block mode that can be used to match against text. +func Compile(expr string) (Database, error) { + db, err := ch.Compile(expr, 0, ch.Groups, nil) + if err != nil { + return nil, err // nolint: wrapcheck + } + + return &database{db}, nil +} + +// MustCompile is like Compile but panics if the expression cannot be parsed. +// It simplifies safe initialization of global variables holding compiled regular expressions. +func MustCompile(expr string) Database { + db, err := Compile(expr) + if err != nil { + panic(`Compile(` + Quote(expr) + `): ` + err.Error()) + } + + return db +} + +// Quote returns a quoted string literal representing s. +func Quote(s string) string { + if strconv.CanBackquote(s) { + return "`" + s + "`" + } + + return strconv.Quote(s) +} diff --git a/chimera/doc.go b/chimera/doc.go new file mode 100644 index 0000000..f3898c3 --- /dev/null +++ b/chimera/doc.go @@ -0,0 +1,56 @@ +// Chimera is a software regular expression matching engine that is a hybrid of Hyperscan and PCRE. +// The design goals of Chimera are to fully support PCRE syntax as well as to +// take advantage of the high performance nature of Hyperscan. +// +// Chimera inherits the design guideline of Hyperscan with C APIs for compilation and scanning. +// +// The Chimera API itself is composed of two major components: +// +// Compilation +// +// These functions take a group of regular expressions, along with identifiers and option flags, +// and compile them into an immutable database that can be used by the Chimera scanning API. +// This compilation process performs considerable analysis and optimization work in order to build a database +// that will match the given expressions efficiently. +// +// See Compiling Patterns for more details (https://intel.github.io/hyperscan/dev-reference/chimera.html#chcompile) +// +// Scanning +// +// Once a Chimera database has been created, it can be used to scan data in memory. +// Chimera only supports block mode in which we scan a single contiguous block in memory. +// +// Matches are delivered to the application via a user-supplied callback function +// that is called synchronously for each match. +// +// For a given database, Chimera provides several guarantees: +// +// 1 No memory allocations occur at runtime with the exception of scratch space allocation, +// it should be done ahead of time for performance-critical applications: +// +// 2 Scratch space: temporary memory used for internal data at scan time. +// Structures in scratch space do not persist beyond the end of a single scan call. +// +// 3 The size of the scratch space required for a given database is fixed and determined at database compile time. +// This means that the memory requirement of the application are known ahead of time, +// and the scratch space can be pre-allocated if required for performance reasons. +// +// 4 Any pattern that has successfully been compiled by the Chimera compiler can be scanned against any input. +// There could be internal resource limits or other limitations caused by PCRE at runtime +// that could cause a scan call to return an error. +// +// * Note +// +// Chimera is designed to have the same matching behavior as PCRE, including greedy/ungreedy, capturing, etc. +// Chimera reports both start offset and end offset for each match like PCRE. +// Different from the fashion of reporting all matches in Hyperscan, Chimera only reports non-overlapping matches. +// For example, the pattern /foofoo/ will match foofoofoofoo at offsets (0, 6) and (6, 12). +// +// * Note +// +// Since Chimera is a hybrid of Hyperscan and PCRE in order to support full PCRE syntax, +// there will be extra performance overhead compared to Hyperscan-only solution. +// Please always use Hyperscan for better performance unless you must need full PCRE syntax support. +// +// See Scanning for Patterns for more details (https://intel.github.io/hyperscan/dev-reference/chimera.html#chruntime) +package chimera diff --git a/chimera/pattern.go b/chimera/pattern.go new file mode 100644 index 0000000..8e6c07a --- /dev/null +++ b/chimera/pattern.go @@ -0,0 +1,96 @@ +package chimera + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" + + "github.com/flier/gohs/internal/ch" +) + +// Expression of pattern. +type Expression = ch.Expression + +// Pattern is a matching pattern. +type Pattern = ch.Pattern + +// NewPattern returns a new pattern base on expression and compile flags. +func NewPattern(expr string, flags CompileFlag) *Pattern { + return &Pattern{Expression: Expression(expr), Flags: flags} +} + +/* +ParsePattern parse pattern from a formated string. + + :// + +For example, the following pattern will match `test` in the caseless and multi-lines mode + + /test/im + +*/ +func ParsePattern(s string) (*Pattern, error) { + var p Pattern + + i := strings.Index(s, ":/") + j := strings.LastIndex(s, "/") + + if i > 0 && j > i+1 { + id, err := strconv.Atoi(s[:i]) + if err != nil { + return nil, fmt.Errorf("pattern id `%s`, %w", s[:i], ErrInvalid) + } + + p.Id = id + s = s[i+1:] + } + + if n := strings.LastIndex(s, "/"); n > 1 && strings.HasPrefix(s, "/") { + p.Expression = Expression(s[1:n]) + s = s[n+1:] + + flags, err := ParseCompileFlag(s) + if err != nil { + return nil, fmt.Errorf("pattern flags `%s`, %w", s, err) + } + + p.Flags = flags + } else { + p.Expression = Expression(s) + } + + return &p, nil +} + +// Patterns is a set of matching patterns. +type Patterns []*Pattern + +// ParsePatterns parse lines as `Patterns`. +func ParsePatterns(r io.Reader) (patterns Patterns, err error) { + s := bufio.NewScanner(r) + + for s.Scan() { + line := strings.TrimSpace(s.Text()) + + if line == "" { + // skip empty line + continue + } + + if strings.HasPrefix(line, "#") { + // skip comment + continue + } + + p, err := ParsePattern(line) + if err != nil { + return nil, err + } + + patterns = append(patterns, p) + } + + return +} diff --git a/chimera/pattern_test.go b/chimera/pattern_test.go new file mode 100644 index 0000000..2e87fc0 --- /dev/null +++ b/chimera/pattern_test.go @@ -0,0 +1,82 @@ +package chimera_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/chimera" +) + +//nolint:funlen +func TestPattern(t *testing.T) { + Convey("Give a pattern", t, func() { + Convey("When parse with flags", func() { + p, err := chimera.ParsePattern(`/test/im`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "test") + So(p.Flags, ShouldEqual, chimera.Caseless|chimera.MultiLine) + + So(string(p.Expression), ShouldEqual, "test") + So(p.String(), ShouldEqual, `/test/im`) + + Convey("When pattern contains forward slash", func() { + p, err := chimera.ParsePattern(`/te/st/im`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "te/st") + So(p.Flags, ShouldEqual, chimera.Caseless|chimera.MultiLine) + + So(p.String(), ShouldEqual, "/te/st/im") + }) + }) + + Convey("When parse pattern with id and flags", func() { + p, err := chimera.ParsePattern("3:/foobar/i8") + + So(err, ShouldBeNil) + So(p.Id, ShouldEqual, 3) + So(p.Expression, ShouldEqual, "foobar") + So(p.Flags, ShouldEqual, chimera.Caseless|chimera.Utf8Mode) + }) + + Convey("When parse with a lot of flags", func() { + p, err := chimera.ParsePattern(`/test/ismH8W`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "test") + So(p.Flags, ShouldEqual, chimera.Caseless|chimera.DotAll|chimera.MultiLine|chimera.SingleMatch| + chimera.Utf8Mode|chimera.UnicodeProperty) + + So(p.Flags.String(), ShouldEqual, "8HWims") + So(p.String(), ShouldEqual, "/test/8HWims") + }) + + Convey("When parse without flags", func() { + p, err := chimera.ParsePattern(`test`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "test") + So(p.Flags, ShouldEqual, 0) + + Convey("When pattern contains forward slash", func() { + p, err := chimera.ParsePattern(`te/st`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "te/st") + So(p.Flags, ShouldEqual, 0) + }) + }) + + Convey("When quote a string", func() { + So(chimera.Quote("test"), ShouldEqual, "`test`") + So(chimera.Quote("`can't backquote this`"), ShouldEqual, "\"`can't backquote this`\"") + }) + }) +} diff --git a/chimera/runtime.go b/chimera/runtime.go new file mode 100644 index 0000000..dd12e82 --- /dev/null +++ b/chimera/runtime.go @@ -0,0 +1,6 @@ +package chimera + +import "github.com/flier/gohs/internal/ch" + +// A Chimera scratch space. +type Scratch = ch.Scratch diff --git a/examples/simplegrep/main.go b/examples/simplegrep/main.go index 9cbe07a..42954fd 100644 --- a/examples/simplegrep/main.go +++ b/examples/simplegrep/main.go @@ -93,7 +93,7 @@ func main() { } } - pattern := hyperscan.NewPattern(flag.Arg(0), hyperscan.DotAll|hyperscan.SomLeftMost) + pattern := hyperscan.NewPattern(hyperscan.Expression(flag.Arg(0)), hyperscan.DotAll|hyperscan.SomLeftMost) inputFN := flag.Arg(1) /* First, we attempt to compile the pattern provided on the command line. diff --git a/hyperscan/api.go b/hyperscan/api.go index c3f7211..f478469 100644 --- a/hyperscan/api.go +++ b/hyperscan/api.go @@ -3,44 +3,9 @@ package hyperscan import ( "fmt" "io" -) - -type matchEvent struct { - id uint - from, to uint64 - flags ScanFlag -} - -func (e *matchEvent) Id() uint { return e.id } // nolint: golint,revive,stylecheck - -func (e *matchEvent) From() uint64 { return e.from } - -func (e *matchEvent) To() uint64 { return e.to } - -func (e *matchEvent) Flags() ScanFlag { return e.flags } - -type matchRecorder struct { - matched []matchEvent - err error -} -func (h *matchRecorder) Matched() bool { return h.matched != nil } - -func (h *matchRecorder) Handle(id uint, from, to uint64, flags uint, context interface{}) error { - if len(h.matched) > 0 { - tail := &h.matched[len(h.matched)-1] - - if tail.id == id && tail.from == from && tail.flags == ScanFlag(flags) && tail.to < to { - tail.to = to - - return h.err - } - } - - h.matched = append(h.matched, matchEvent{id, from, to, ScanFlag(flags)}) - - return h.err -} + "github.com/flier/gohs/internal/hs" +) // Match reports whether the byte slice b contains any match of the regular expression pattern. func Match(pattern string, data []byte) (bool, error) { @@ -66,13 +31,13 @@ func Match(pattern string, data []byte) (bool, error) { _ = s.Free() }() - h := &matchRecorder{} + h := &hs.MatchRecorder{} if err = db.Scan(data, s, h.Handle, nil); err != nil { return false, err // nolint: wrapcheck } - return h.Matched(), h.err + return h.Matched(), h.Err } // MatchReader reports whether the text returned by the Reader contains any match of the regular expression pattern. @@ -99,13 +64,13 @@ func MatchReader(pattern string, reader io.Reader) (bool, error) { _ = s.Free() }() - h := &matchRecorder{} + h := &hs.MatchRecorder{} if err = db.Scan(reader, s, h.Handle, nil); err != nil { return false, err // nolint: wrapcheck } - return h.Matched(), h.err + return h.Matched(), h.Err } // MatchString reports whether the string s contains any match of the regular expression pattern. diff --git a/hyperscan/block.go b/hyperscan/block.go new file mode 100644 index 0000000..651f19c --- /dev/null +++ b/hyperscan/block.go @@ -0,0 +1,194 @@ +package hyperscan + +import ( + "errors" + + "github.com/flier/gohs/internal/hs" +) + +// BlockScanner is the block (non-streaming) regular expression scanner. +type BlockScanner interface { + // This is the function call in which the actual pattern matching takes place for block-mode pattern databases. + Scan(data []byte, scratch *Scratch, handler MatchHandler, context interface{}) error +} + +// BlockMatcher implements regular expression search. +type BlockMatcher interface { + // Find returns a slice holding the text of the leftmost match in b of the regular expression. + // A return value of nil indicates no match. + Find(data []byte) []byte + + // FindIndex returns a two-element slice of integers defining + // the location of the leftmost match in b of the regular expression. + // The match itself is at b[loc[0]:loc[1]]. A return value of nil indicates no match. + FindIndex(data []byte) []int + + // FindAll is the 'All' version of Find; it returns a slice of all successive matches of the expression, + // as defined by the 'All' description in the package comment. A return value of nil indicates no match. + FindAll(data []byte, n int) [][]byte + + // FindAllIndex is the 'All' version of FindIndex; it returns a slice of all successive matches of the expression, + // as defined by the 'All' description in the package comment. A return value of nil indicates no match. + FindAllIndex(data []byte, n int) [][]int + + // FindString returns a string holding the text of the leftmost match in s of the regular expression. + // If there is no match, the return value is an empty string, but it will also be empty + // if the regular expression successfully matches an empty string. + // Use FindStringIndex if it is necessary to distinguish these cases. + FindString(s string) string + + // FindStringIndex returns a two-element slice of integers defining + // the location of the leftmost match in s of the regular expression. + // The match itself is at s[loc[0]:loc[1]]. A return value of nil indicates no match. + FindStringIndex(s string) []int + + // FindAllString is the 'All' version of FindString; it returns a slice of all successive matches of the expression, + // as defined by the 'All' description in the package comment. A return value of nil indicates no match. + FindAllString(s string, n int) []string + + // FindAllStringIndex is the 'All' version of FindStringIndex; + // it returns a slice of all successive matches of the expression, + // as defined by the 'All' description in the package comment. A return value of nil indicates no match. + FindAllStringIndex(s string, n int) [][]int + + // Match reports whether the pattern database matches the byte slice b. + Match(b []byte) bool + + // MatchString reports whether the pattern database matches the string s. + MatchString(s string) bool +} + +type blockScanner struct { + *baseDatabase +} + +func newBlockScanner(bdb *baseDatabase) *blockScanner { + return &blockScanner{bdb} +} + +func (bs *blockScanner) Scan(data []byte, s *Scratch, handler MatchHandler, context interface{}) (err error) { + if s == nil { + s, err = NewScratch(bs) + + if err != nil { + return + } + + defer func() { + _ = s.Free() + }() + } + + return hs.Scan(bs.db, data, 0, s.s, handler, context) // nolint: wrapcheck +} + +type blockMatcher struct { + *blockScanner + *hs.MatchRecorder + n int +} + +func newBlockMatcher(scanner *blockScanner) *blockMatcher { + return &blockMatcher{blockScanner: scanner} +} + +func (m *blockMatcher) Handle(id uint, from, to uint64, flags uint, context interface{}) error { + err := m.MatchRecorder.Handle(id, from, to, flags, context) + if err != nil { + return err // nolint: wrapcheck + } + + if m.n < 0 { + return nil + } + + if m.n < len(m.Events) { + m.Events = m.Events[:m.n] + + return ErrTooManyMatches + } + + return nil +} + +func (m *blockMatcher) scan(data []byte) error { + m.MatchRecorder = &hs.MatchRecorder{} + + return m.blockScanner.Scan(data, nil, m.Handle, nil) +} + +const findIndexMatches = 2 + +func (m *blockMatcher) Find(data []byte) []byte { + if loc := m.FindIndex(data); len(loc) == findIndexMatches { + return data[loc[0]:loc[1]] + } + + return nil +} + +func (m *blockMatcher) FindIndex(data []byte) []int { + if m.Match(data) && len(m.Events) == 1 { + return []int{int(m.Events[0].From), int(m.Events[0].To)} + } + + return nil +} + +func (m *blockMatcher) FindAll(data []byte, n int) (matches [][]byte) { + if locs := m.FindAllIndex(data, n); len(locs) > 0 { + for _, loc := range locs { + matches = append(matches, data[loc[0]:loc[1]]) + } + } + + return +} + +func (m *blockMatcher) FindAllIndex(data []byte, n int) (locs [][]int) { + if n < 0 { + n = len(data) + 1 + } + + m.n = n + + if err := m.scan(data); (err == nil || errors.Is(err, ErrScanTerminated)) && len(m.Events) > 0 { + for _, e := range m.Events { + locs = append(locs, []int{int(e.From), int(e.To)}) + } + } + + return +} + +func (m *blockMatcher) FindString(s string) string { + return string(m.Find([]byte(s))) +} + +func (m *blockMatcher) FindStringIndex(s string) (loc []int) { + return m.FindIndex([]byte(s)) +} + +func (m *blockMatcher) FindAllString(s string, n int) (results []string) { + for _, m := range m.FindAll([]byte(s), n) { + results = append(results, string(m)) + } + + return +} + +func (m *blockMatcher) FindAllStringIndex(s string, n int) [][]int { + return m.FindAllIndex([]byte(s), n) +} + +func (m *blockMatcher) Match(data []byte) bool { + m.n = 1 + + err := m.scan(data) + + return (err == nil || errors.Is(err, ErrScanTerminated)) && len(m.Events) == m.n +} + +func (m *blockMatcher) MatchString(s string) bool { + return m.Match([]byte(s)) +} diff --git a/hyperscan/common.go b/hyperscan/common.go index 0a2fea1..674b996 100644 --- a/hyperscan/common.go +++ b/hyperscan/common.go @@ -3,6 +3,39 @@ package hyperscan import ( "fmt" "regexp" + + "github.com/flier/gohs/internal/hs" +) + +type HsError = hs.Error + +const ( + // ErrSuccess is the error returned if the engine completed normally. + ErrSuccess HsError = hs.ErrSuccess + // ErrInvalid is the error returned if a parameter passed to this function was invalid. + ErrInvalid HsError = hs.ErrInvalid + // ErrNoMemory is the error returned if a memory allocation failed. + ErrNoMemory HsError = hs.ErrNoMemory + // ErrScanTerminated is the error returned if the engine was terminated by callback. + ErrScanTerminated HsError = hs.ErrScanTerminated + // ErrCompileError is the error returned if the pattern compiler failed. + ErrCompileError HsError = hs.ErrCompileError + // ErrDatabaseVersionError is the error returned if the given database was built for a different version of Hyperscan. + ErrDatabaseVersionError HsError = hs.ErrDatabaseVersionError + // ErrDatabasePlatformError is the error returned if the given database was built for a different platform. + ErrDatabasePlatformError HsError = hs.ErrDatabasePlatformError + // ErrDatabaseModeError is the error returned if the given database was built for a different mode of operation. + ErrDatabaseModeError HsError = hs.ErrDatabaseModeError + // ErrBadAlign is the error returned if a parameter passed to this function was not correctly aligned. + ErrBadAlign HsError = hs.ErrBadAlign + // ErrBadAlloc is the error returned if the memory allocator did not correctly return memory suitably aligned. + ErrBadAlloc HsError = hs.ErrBadAlloc + // ErrScratchInUse is the error returned if the scratch region was already in use. + ErrScratchInUse HsError = hs.ErrScratchInUse + // ErrArchError is the error returned if unsupported CPU architecture. + ErrArchError HsError = hs.ErrArchError + // ErrInsufficientSpace is the error returned if provided buffer was too small. + ErrInsufficientSpace HsError = hs.ErrInsufficientSpace ) // Database is an immutable database that can be used by the Hyperscan scanning API. @@ -84,28 +117,28 @@ func (i DbInfo) Mode() (ModeFlag, error) { // Version identify this release version. The return version is a string // containing the version number of this release build and the date of the build. -func Version() string { return hsVersion() } +func Version() string { return hs.Version() } // ValidPlatform test the current system architecture. -func ValidPlatform() error { return hsValidPlatform() } +func ValidPlatform() error { return hs.ValidPlatform() } // nolint: wrapcheck type database interface { - Db() hsDatabase + Db() hs.Database } type baseDatabase struct { - db hsDatabase + db hs.Database } -func newBaseDatabase(db hsDatabase) *baseDatabase { +func newBaseDatabase(db hs.Database) *baseDatabase { return &baseDatabase{db} } // UnmarshalDatabase reconstruct a pattern database from a stream of bytes. func UnmarshalDatabase(data []byte) (Database, error) { - db, err := hsDeserializeDatabase(data) + db, err := hs.DeserializeDatabase(data) if err != nil { - return nil, err + return nil, err // nolint: wrapcheck } return &baseDatabase{db}, nil @@ -113,9 +146,9 @@ func UnmarshalDatabase(data []byte) (Database, error) { // UnmarshalBlockDatabase reconstruct a block database from a stream of bytes. func UnmarshalBlockDatabase(data []byte) (BlockDatabase, error) { - db, err := hsDeserializeDatabase(data) + db, err := hs.DeserializeDatabase(data) if err != nil { - return nil, err + return nil, err // nolint: wrapcheck } return newBlockDatabase(db), nil @@ -123,9 +156,9 @@ func UnmarshalBlockDatabase(data []byte) (BlockDatabase, error) { // UnmarshalStreamDatabase reconstruct a stream database from a stream of bytes. func UnmarshalStreamDatabase(data []byte) (StreamDatabase, error) { - db, err := hsDeserializeDatabase(data) + db, err := hs.DeserializeDatabase(data) if err != nil { - return nil, err + return nil, err // nolint: wrapcheck } return newStreamDatabase(db), nil @@ -133,45 +166,48 @@ func UnmarshalStreamDatabase(data []byte) (StreamDatabase, error) { // UnmarshalVectoredDatabase reconstruct a vectored database from a stream of bytes. func UnmarshalVectoredDatabase(data []byte) (VectoredDatabase, error) { - db, err := hsDeserializeDatabase(data) + db, err := hs.DeserializeDatabase(data) if err != nil { - return nil, err + return nil, err // nolint: wrapcheck } return newVectoredDatabase(db), nil } // SerializedDatabaseSize reports the size that would be required by a database if it were deserialized. -func SerializedDatabaseSize(data []byte) (int, error) { return hsSerializedDatabaseSize(data) } +func SerializedDatabaseSize(data []byte) (int, error) { return hs.SerializedDatabaseSize(data) } // nolint: wrapcheck // SerializedDatabaseInfo provides information about a serialized database. func SerializedDatabaseInfo(data []byte) (DbInfo, error) { - i, err := hsSerializedDatabaseInfo(data) + i, err := hs.SerializedDatabaseInfo(data) return DbInfo(i), err } -func (d *baseDatabase) Db() hsDatabase { return d.db } // nolint: stylecheck +func (d *baseDatabase) Db() hs.Database { return d.db } // nolint: stylecheck -func (d *baseDatabase) Size() (int, error) { return hsDatabaseSize(d.db) } +func (d *baseDatabase) Size() (int, error) { return hs.DatabaseSize(d.db) } // nolint: wrapcheck func (d *baseDatabase) Info() (DbInfo, error) { - i, err := hsDatabaseInfo(d.db) + i, err := hs.DatabaseInfo(d.db) + if err != nil { + return "", err //nolint: wrapcheck + } - return DbInfo(i), err + return DbInfo(i), nil } -func (d *baseDatabase) Close() error { return hsFreeDatabase(d.db) } +func (d *baseDatabase) Close() error { return hs.FreeDatabase(d.db) } // nolint: wrapcheck -func (d *baseDatabase) Marshal() ([]byte, error) { return hsSerializeDatabase(d.db) } +func (d *baseDatabase) Marshal() ([]byte, error) { return hs.SerializeDatabase(d.db) } // nolint: wrapcheck -func (d *baseDatabase) Unmarshal(data []byte) error { return hsDeserializeDatabaseAt(data, d.db) } +func (d *baseDatabase) Unmarshal(data []byte) error { return hs.DeserializeDatabaseAt(data, d.db) } // nolint: wrapcheck type blockDatabase struct { *blockMatcher } -func newBlockDatabase(db hsDatabase) *blockDatabase { +func newBlockDatabase(db hs.Database) *blockDatabase { return &blockDatabase{newBlockMatcher(newBlockScanner(newBaseDatabase(db)))} } @@ -179,16 +215,16 @@ type streamDatabase struct { *streamMatcher } -func newStreamDatabase(db hsDatabase) *streamDatabase { +func newStreamDatabase(db hs.Database) *streamDatabase { return &streamDatabase{newStreamMatcher(newStreamScanner(newBaseDatabase(db)))} } -func (db *streamDatabase) StreamSize() (int, error) { return hsStreamSize(db.db) } +func (db *streamDatabase) StreamSize() (int, error) { return hs.StreamSize(db.db) } // nolint: wrapcheck type vectoredDatabase struct { *vectoredMatcher } -func newVectoredDatabase(db hsDatabase) *vectoredDatabase { +func newVectoredDatabase(db hs.Database) *vectoredDatabase { return &vectoredDatabase{newVectoredMatcher(newVectoredScanner(newBaseDatabase(db)))} } diff --git a/hyperscan/compile.go b/hyperscan/compile.go index c312f76..94afb91 100644 --- a/hyperscan/compile.go +++ b/hyperscan/compile.go @@ -1,217 +1,101 @@ package hyperscan import ( - "bufio" - "errors" "fmt" - "io" "runtime" "strconv" "strings" -) -var ( - // ErrNoFound means patterns not found. - ErrNoFound = errors.New("no found") - // ErrUnexpected means item is unexpected. - ErrUnexpected = errors.New("unexpected") + "github.com/flier/gohs/internal/hs" ) -// Expression of pattern. -type Expression string - -func (e Expression) String() string { return string(e) } - -// Patterns is a set of matching patterns. -type Patterns []*Pattern - -// Pattern is a matching pattern. -// nolint: golint,revive,stylecheck -type Pattern struct { - Expression // The expression to parse. - Flags CompileFlag // Flags which modify the behaviour of the expression. - Id int // The ID number to be associated with the corresponding pattern - info *ExprInfo - ext *ExprExt -} - -// NewPattern returns a new pattern base on expression and compile flags. -func NewPattern(expr string, flags ...CompileFlag) *Pattern { - var v CompileFlag - for _, f := range flags { - v |= f - } - return &Pattern{Expression: Expression(expr), Flags: v} -} - -// IsValid validate the pattern contains a regular expression. -func (p *Pattern) IsValid() bool { - _, err := p.Info() - - return err == nil -} - -// Info provides information about a regular expression. -func (p *Pattern) Info() (*ExprInfo, error) { - if p.info == nil { - info, err := hsExpressionInfo(string(p.Expression), p.Flags) - if err != nil { - return nil, err - } - - p.info = info - } - - return p.info, nil -} - -// WithExt is used to set the additional parameters related to an expression. -func (p *Pattern) WithExt(exts ...Ext) *Pattern { - if p.ext == nil { - p.ext = new(ExprExt) - } - - p.ext.With(exts...) - - return p -} - -// Ext provides additional parameters related to an expression. -func (p *Pattern) Ext() (*ExprExt, error) { - if p.ext == nil { - ext, info, err := hsExpressionExt(string(p.Expression), p.Flags) - if err != nil { - return nil, err - } - - p.ext = ext - p.info = info - } - - return p.ext, nil -} - -func (p *Pattern) String() string { - var b strings.Builder - - if p.Id > 0 { - fmt.Fprintf(&b, "%d:", p.Id) - } - - fmt.Fprintf(&b, "/%s/%s", p.Expression, p.Flags) - - if p.ext != nil { - b.WriteString(p.ext.String()) - } - - return b.String() -} +// A type containing error details that is returned by the compile calls on failure. +// +// The caller may inspect the values returned in this type to determine the cause of failure. +type CompileError = hs.CompileError + +// CompileFlag represents a pattern flag. +type CompileFlag = hs.CompileFlag + +const ( + // Caseless represents set case-insensitive matching. + Caseless CompileFlag = hs.Caseless + // DotAll represents matching a `.` will not exclude newlines. + DotAll CompileFlag = hs.DotAll + // MultiLine set multi-line anchoring. + MultiLine CompileFlag = hs.MultiLine + // SingleMatch set single-match only mode. + SingleMatch CompileFlag = hs.SingleMatch + // AllowEmpty allow expressions that can match against empty buffers. + AllowEmpty CompileFlag = hs.AllowEmpty + // Utf8Mode enable UTF-8 mode for this expression. + Utf8Mode CompileFlag = hs.Utf8Mode + // UnicodeProperty enable Unicode property support for this expression. + UnicodeProperty CompileFlag = hs.UnicodeProperty + // PrefilterMode enable prefiltering mode for this expression. + PrefilterMode CompileFlag = hs.PrefilterMode + // SomLeftMost enable leftmost start of match reporting. + SomLeftMost CompileFlag = hs.SomLeftMost +) /* -ParsePattern parse pattern from a formated string. - - :// - -For example, the following pattern will match `test` in the caseless and multi-lines mode - - /test/im - +ParseCompileFlag parse the compile pattern flags from string + + i Caseless Case-insensitive matching + s DotAll Dot (.) will match newlines + m MultiLine Multi-line anchoring + H SingleMatch Report match ID at most once (`o` deprecated) + V AllowEmpty Allow patterns that can match against empty buffers (`e` deprecated) + 8 Utf8Mode UTF-8 mode (`u` deprecated) + W UnicodeProperty Unicode property support (`p` deprecated) + P PrefilterMode Prefiltering mode (`f` deprecated) + L SomLeftMost Leftmost start of match reporting (`l` deprecated) + C Combination Logical combination of patterns (Hyperscan 5.0) + Q Quiet Quiet at matching (Hyperscan 5.0) */ -func ParsePattern(s string) (*Pattern, error) { - var p Pattern - - i := strings.Index(s, ":/") - j := strings.LastIndex(s, "/") - - if i > 0 && j > i+1 { - id, err := strconv.Atoi(s[:i]) - if err != nil { - return nil, fmt.Errorf("invalid pattern id `%s`, %w", s[:i], ErrInvalid) +func ParseCompileFlag(s string) (CompileFlag, error) { + var flags CompileFlag + + for _, c := range s { + if flag, exists := hs.CompileFlags[c]; exists { + flags |= flag + } else if flag, exists := hs.DeprecatedCompileFlags[c]; exists { + flags |= flag + } else { + return 0, fmt.Errorf("flag `%c`, %w", c, ErrInvalid) } - - p.Id = id - s = s[i+1:] - } - - if n := strings.LastIndex(s, "/"); n > 1 && strings.HasPrefix(s, "/") { - p.Expression = Expression(s[1:n]) - s = s[n+1:] - - if n = strings.Index(s, "{"); n > 0 && strings.HasSuffix(s, "}") { - ext, err := ParseExprExt(s[n:]) - if err != nil { - return nil, fmt.Errorf("invalid expression extensions `%s`, %w", s[n:], err) - } - - p.ext = ext - s = s[:n] - } - - flags, err := ParseCompileFlag(s) - if err != nil { - return nil, fmt.Errorf("invalid pattern flags `%s`, %w", s, err) - } - - p.Flags = flags - } else { - p.Expression = Expression(s) - } - - info, err := hsExpressionInfo(string(p.Expression), p.Flags) - if err != nil { - return nil, fmt.Errorf("invalid pattern `%s`, %w", p.Expression, err) } - p.info = info - - return &p, nil + return flags, nil } -// ParsePatterns parse lines as `Patterns`. -func ParsePatterns(r io.Reader) (patterns Patterns, err error) { - s := bufio.NewScanner(r) - - for s.Scan() { - line := strings.TrimSpace(s.Text()) - - if line == "" { - // skip empty line - continue - } - - if strings.HasPrefix(line, "#") { - // skip comment - continue - } - - p, err := ParsePattern(line) - if err != nil { - return nil, err - } +// ModeFlag represents the compile mode flags. +type ModeFlag = hs.ModeFlag + +const ( + // BlockMode for the block scan (non-streaming) database. + BlockMode ModeFlag = hs.BlockMode + // NoStreamMode is alias for Block. + NoStreamMode ModeFlag = hs.NoStreamMode + // StreamMode for the streaming database. + StreamMode ModeFlag = hs.StreamMode + // VectoredMode for the vectored scanning database. + VectoredMode ModeFlag = hs.VectoredMode + // SomHorizonLargeMode use full precision to track start of match offsets in stream state. + SomHorizonLargeMode ModeFlag = hs.SomHorizonLargeMode + // SomHorizonMediumMode use medium precision to track start of match offsets in stream state (within 2^32 bytes). + SomHorizonMediumMode ModeFlag = hs.SomHorizonMediumMode + // SomHorizonSmallMode use limited precision to track start of match offsets in stream state (within 2^16 bytes). + SomHorizonSmallMode ModeFlag = hs.SomHorizonSmallMode +) - patterns = append(patterns, p) +// ParseModeFlag parse a database mode from string. +func ParseModeFlag(s string) (ModeFlag, error) { + if mode, exists := hs.ModeFlags[strings.ToUpper(s)]; exists { + return mode, nil } - return -} - -// Platform is a type containing information on the target platform. -type Platform interface { - // Information about the target platform which may be used to guide the optimisation process of the compile. - Tune() TuneFlag - - // Relevant CPU features available on the target platform - CpuFeatures() CpuFeature -} - -// NewPlatform create a new platform information on the target platform. -func NewPlatform(tune TuneFlag, cpu CpuFeature) Platform { return newPlatformInfo(tune, cpu) } - -// PopulatePlatform populates the platform information based on the current host. -func PopulatePlatform() Platform { - platform, _ := hsPopulatePlatform() - - return platform + return BlockMode, fmt.Errorf("database mode %s, %w", s, ErrInvalid) } type Builder interface { @@ -225,7 +109,7 @@ func (p *Pattern) Build(mode ModeFlag) (Database, error) { } func (p *Pattern) ForPlatform(mode ModeFlag, platform Platform) (Database, error) { - b := DatabaseBuilder{Patterns: []*Pattern{p}, Mode: mode, Platform: platform} + b := DatabaseBuilder{Patterns: Patterns{p}, Mode: mode, Platform: platform} return b.Build() } @@ -241,7 +125,7 @@ func (p Patterns) ForPlatform(mode ModeFlag, platform Platform) (Database, error // DatabaseBuilder to help to build up a database. type DatabaseBuilder struct { // Array of patterns to compile. - Patterns []*Pattern + Patterns // Compiler mode flags that affect the database as a whole. (Default: block mode) Mode ModeFlag @@ -270,7 +154,7 @@ func (b *DatabaseBuilder) AddExpressionWithFlags(expr Expression, flags CompileF // Build a database base on the expressions and platform. func (b *DatabaseBuilder) Build() (Database, error) { if b.Patterns == nil { - return nil, ErrNoFound + return nil, ErrInvalid } mode := b.Mode @@ -291,14 +175,14 @@ func (b *DatabaseBuilder) Build() (Database, error) { } } - platform, _ := b.Platform.(*hsPlatformInfo) + platform, _ := b.Platform.(*hs.PlatformInfo) - db, err := hsCompileMulti(b.Patterns, mode, platform) + db, err := hs.CompileMulti(b.Patterns, mode, platform) if err != nil { - return nil, err + return nil, err // nolint: wrapcheck } - switch mode & ModeMask { + switch mode & hs.ModeMask { case StreamMode: return newStreamDatabase(db), nil case VectoredMode: @@ -306,7 +190,7 @@ func (b *DatabaseBuilder) Build() (Database, error) { case BlockMode: return newBlockDatabase(db), nil default: - return nil, fmt.Errorf("mode %d, %w", mode, ErrUnexpected) + return nil, fmt.Errorf("mode %d, %w", mode, ErrInvalid) } } @@ -395,9 +279,9 @@ func NewVectoredDatabase(patterns ...*Pattern) (VectoredDatabase, error) { // Compile a regular expression and returns, if successful, // a pattern database in the block mode that can be used to match against text. func Compile(expr string) (Database, error) { - db, err := hsCompile(expr, SomLeftMost, BlockMode, nil) + db, err := hs.Compile(expr, SomLeftMost, BlockMode, nil) if err != nil { - return nil, err + return nil, err // nolint: wrapcheck } return newBlockDatabase(db), nil @@ -406,12 +290,12 @@ func Compile(expr string) (Database, error) { // MustCompile is like Compile but panics if the expression cannot be parsed. // It simplifies safe initialization of global variables holding compiled regular expressions. func MustCompile(expr string) Database { - db, err := hsCompile(expr, SomLeftMost, BlockMode, nil) + db, err := Compile(expr) if err != nil { panic(`Compile(` + Quote(expr) + `): ` + err.Error()) } - return newBlockDatabase(db) + return db } // Quote returns a quoted string literal representing s. diff --git a/hyperscan/compile_test.go b/hyperscan/compile_test.go index cbba97b..a943514 100644 --- a/hyperscan/compile_test.go +++ b/hyperscan/compile_test.go @@ -9,109 +9,47 @@ import ( "github.com/flier/gohs/hyperscan" ) -func TestPattern(t *testing.T) { - Convey("Give a pattern", t, func() { - Convey("When parse with flags", func() { - p, err := hyperscan.ParsePattern(`/test/im`) +func TestCompileFlag(t *testing.T) { + Convey("Given a compile flags", t, func() { + flags := hyperscan.Caseless | hyperscan.DotAll | hyperscan.MultiLine | hyperscan.SingleMatch | + hyperscan.AllowEmpty | hyperscan.Utf8Mode | hyperscan.UnicodeProperty | hyperscan.PrefilterMode - So(err, ShouldBeNil) - So(p, ShouldNotBeNil) - So(p.Expression, ShouldEqual, "test") - So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) - - So(p.Expression.String(), ShouldEqual, "test") - So(p.String(), ShouldEqual, `/test/im`) - - Convey("When pattern contains forward slash", func() { - p, err := hyperscan.ParsePattern(`/te/st/im`) - - So(err, ShouldBeNil) - So(p, ShouldNotBeNil) - So(p.Expression, ShouldEqual, "te/st") - So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) - - So(p.String(), ShouldEqual, "/te/st/im") - }) - }) - - Convey("When parse pattern with id and flags", func() { - p, err := hyperscan.ParsePattern("3:/foobar/iu") - - So(err, ShouldBeNil) - So(p.Id, ShouldEqual, 3) - So(p.Expression, ShouldEqual, "foobar") - So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.Utf8Mode) - }) - - Convey("When parse pattern with id, flags and extensions", func() { - p, err := hyperscan.ParsePattern("3:/foobar/iu{min_offset=4,min_length=8}") - So(err, ShouldBeNil) - So(p.Id, ShouldEqual, 3) - So(p.Expression, ShouldEqual, "foobar") - So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.Utf8Mode) - - ext, err := p.Ext() - So(err, ShouldBeNil) - So(ext, ShouldResemble, new(hyperscan.ExprExt).With(hyperscan.MinOffset(4), hyperscan.MinLength(8))) - - So(p.String(), ShouldEqual, "3:/foobar/8i{min_offset=4,min_length=8}") - }) + So(flags.String(), ShouldEqual, "8HPVWims") - Convey("When parse with a lot of flags", func() { - p, err := hyperscan.ParsePattern(`/test/ismoeupf`) + Convey("When parse valid flags", func() { + f, err := hyperscan.ParseCompileFlag("ifemopus") + So(f, ShouldEqual, flags) So(err, ShouldBeNil) - So(p, ShouldNotBeNil) - So(p.Expression, ShouldEqual, "test") - So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.DotAll|hyperscan.MultiLine|hyperscan.SingleMatch| - hyperscan.AllowEmpty|hyperscan.Utf8Mode|hyperscan.UnicodeProperty|hyperscan.PrefilterMode) - - So(p.Flags.String(), ShouldEqual, "8HPVWims") - So(p.String(), ShouldEqual, "/test/8HPVWims") }) - Convey("When parse without flags", func() { - p, err := hyperscan.ParsePattern(`test`) + Convey("When parse invalid flags", func() { + f, err := hyperscan.ParseCompileFlag("abc") - So(err, ShouldBeNil) - So(p, ShouldNotBeNil) - So(p.Expression, ShouldEqual, "test") - So(p.Flags, ShouldEqual, 0) - - Convey("When pattern contains forward slash", func() { - p, err := hyperscan.ParsePattern(`te/st`) - - So(err, ShouldBeNil) - So(p, ShouldNotBeNil) - So(p.Expression, ShouldEqual, "te/st") - So(p.Flags, ShouldEqual, 0) - }) + So(f, ShouldEqual, 0) + So(err, ShouldNotBeNil) }) + }) +} - Convey("When pattern is valid", func() { - p := hyperscan.Pattern{Expression: "test"} +func TestModeFlag(t *testing.T) { + Convey("Give a mode", t, func() { + So(hyperscan.BlockMode.String(), ShouldEqual, "BLOCK") + So(hyperscan.StreamMode.String(), ShouldEqual, "STREAM") + So(hyperscan.VectoredMode.String(), ShouldEqual, "VECTORED") - info, err := p.Info() + Convey("When combile mode with flags", func() { + mode := hyperscan.StreamMode | hyperscan.SomHorizonLargeMode - So(err, ShouldBeNil) - So(info, ShouldNotBeNil) - So(info, ShouldResemble, &hyperscan.ExprInfo{MinWidth: 4, MaxWidth: 4}) - So(p.IsValid(), ShouldBeTrue) + So(mode.String(), ShouldEqual, "STREAM") }) - Convey("When pattern is invalid", func() { - p := hyperscan.Pattern{Expression: `\R`} - - info, err := p.Info() + Convey("When parse unknown mode", func() { + m, err := hyperscan.ParseModeFlag("test") So(err, ShouldNotBeNil) - So(info, ShouldBeNil) - So(p.IsValid(), ShouldBeFalse) - }) - - Convey("When quote a string", func() { - So(hyperscan.Quote("test"), ShouldEqual, "`test`") - So(hyperscan.Quote("`can't backquote this`"), ShouldEqual, "\"`can't backquote this`\"") + So(err.Error(), ShouldContainSubstring, "database mode test") + So(m, ShouldEqual, hyperscan.BlockMode) }) }) } @@ -217,15 +155,3 @@ func TestCompile(t *testing.T) { }) }) } - -func TestPlatform(t *testing.T) { - Convey("Given a native platform", t, func() { - p := hyperscan.PopulatePlatform() - - So(p, ShouldNotBeNil) - So(p.Tune(), ShouldBeGreaterThan, hyperscan.Generic) - So(p.CpuFeatures(), ShouldBeGreaterThanOrEqualTo, 0) - - So(p, ShouldResemble, hyperscan.NewPlatform(p.Tune(), p.CpuFeatures())) - }) -} diff --git a/hyperscan/compile_v5.go b/hyperscan/compile_v5.go index 0031642..eb11c0b 100644 --- a/hyperscan/compile_v5.go +++ b/hyperscan/compile_v5.go @@ -4,183 +4,12 @@ package hyperscan import ( - "fmt" - "strconv" - "strings" + "github.com/flier/gohs/internal/hs" ) -type Literals []*Literal - -// Pure literal is a special case of regular expression. -// A character sequence is regarded as a pure literal if and -// only if each character is read and interpreted independently. -// No syntax association happens between any adjacent characters. -// nolint: golint,revive,stylecheck -type Literal struct { - Expression // The expression to parse. - Flags CompileFlag // Flags which modify the behaviour of the expression. - Id int // The ID number to be associated with the corresponding pattern - info *ExprInfo -} - -// NewLiteral returns a new Literal base on expression and compile flags. -func NewLiteral(expr string, flags ...CompileFlag) *Literal { - var v CompileFlag - for _, f := range flags { - v |= f - } - return &Literal{Expression: Expression(expr), Flags: v} -} - -// IsValid validate the literal contains a pure literal. -func (lit *Literal) IsValid() bool { - _, err := lit.Info() - - return err == nil -} - -// Provides information about a regular expression. -func (lit *Literal) Info() (*ExprInfo, error) { - if lit.info == nil { - info, err := hsExpressionInfo(string(lit.Expression), lit.Flags) - if err != nil { - return nil, err - } - - lit.info = info - } - - return lit.info, nil -} - -func (lit *Literal) String() string { - var b strings.Builder - - if lit.Id > 0 { - fmt.Fprintf(&b, "%d:", lit.Id) - } - - fmt.Fprintf(&b, "/%s/%s", lit.Expression, lit.Flags) - - return b.String() -} - -/* -Parse literal from a formated string - - :// - -For example, the following literal will match `test` in the caseless and multi-lines mode - - /test/im - -*/ -func ParseLiteral(s string) (*Literal, error) { - var lit Literal - - i := strings.Index(s, ":/") - j := strings.LastIndex(s, "/") - if i > 0 && j > i+1 { - id, err := strconv.Atoi(s[:i]) - if err != nil { - return nil, fmt.Errorf("invalid pattern id `%s`, %w", s[:i], ErrInvalid) - } - lit.Id = id - s = s[i+1:] - } - - if n := strings.LastIndex(s, "/"); n > 1 && strings.HasPrefix(s, "/") { - lit.Expression = Expression(s[1:n]) - s = s[n+1:] - - flags, err := ParseCompileFlag(s) - if err != nil { - return nil, fmt.Errorf("invalid pattern flags `%s`, %w", s, err) - } - lit.Flags = flags - } else { - lit.Expression = Expression(s) - } - - info, err := hsExpressionInfo(string(lit.Expression), lit.Flags) - if err != nil { - return nil, fmt.Errorf("invalid pattern `%s`, %w", lit.Expression, err) - } - lit.info = info - - return &lit, nil -} - -func (lit *Literal) Build(mode ModeFlag) (Database, error) { - return lit.ForPlatform(mode, nil) -} - -func (lit *Literal) ForPlatform(mode ModeFlag, platform Platform) (Database, error) { - if mode == 0 { - mode = BlockMode - } else if mode == StreamMode { - som := (lit.Flags & SomLeftMost) == SomLeftMost - - if som && mode&(SomHorizonSmallMode|SomHorizonMediumMode|SomHorizonLargeMode) == 0 { - mode |= SomHorizonSmallMode - } - } - - p, _ := platform.(*hsPlatformInfo) - - db, err := hsCompileLit(string(lit.Expression), lit.Flags, mode, p) - if err != nil { - return nil, err - } - - switch mode & ModeMask { - case StreamMode: - return newStreamDatabase(db), nil - case VectoredMode: - return newVectoredDatabase(db), nil - case BlockMode: - return newBlockDatabase(db), nil - } - - return nil, fmt.Errorf("mode %d, %w", mode, ErrUnexpected) -} - -func (literals Literals) Build(mode ModeFlag) (Database, error) { - return literals.ForPlatform(mode, nil) -} - -func (literals Literals) ForPlatform(mode ModeFlag, platform Platform) (Database, error) { - if mode == 0 { - mode = BlockMode - } else if mode == StreamMode { - som := false - - for _, lit := range literals { - if (lit.Flags & SomLeftMost) == SomLeftMost { - som = true - } - } - - if som && mode&(SomHorizonSmallMode|SomHorizonMediumMode|SomHorizonLargeMode) == 0 { - mode |= SomHorizonSmallMode - } - } - - p, _ := platform.(*hsPlatformInfo) - - db, err := hsCompileLitMulti(literals, mode, p) - if err != nil { - return nil, err - } - - switch mode & ModeMask { - case StreamMode: - return newStreamDatabase(db), nil - case VectoredMode: - return newVectoredDatabase(db), nil - case BlockMode: - return newBlockDatabase(db), nil - } - - return nil, fmt.Errorf("mode %d, %w", mode, ErrUnexpected) -} +const ( + // Combination represents logical combination. + Combination CompileFlag = hs.Combination + // Quiet represents don't do any match reporting. + Quiet CompileFlag = hs.Quiet +) diff --git a/hyperscan/compile_v5_test.go b/hyperscan/compile_v5_test.go index 32b949a..0ed1295 100644 --- a/hyperscan/compile_v5_test.go +++ b/hyperscan/compile_v5_test.go @@ -11,92 +11,6 @@ import ( "github.com/flier/gohs/hyperscan" ) -//nolint:funlen -func TestLiteral(t *testing.T) { - Convey("Give a literal", t, func() { - Convey("When parse with flags", func() { - p, err := hyperscan.ParseLiteral(`/test/im`) - - So(err, ShouldBeNil) - So(p, ShouldNotBeNil) - So(p.Expression, ShouldEqual, "test") - So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) - - So(p.Expression.String(), ShouldEqual, "test") - So(p.String(), ShouldEqual, `/test/im`) - - Convey("When literal contains regular grammar", func() { - p, err := hyperscan.ParseLiteral(`/te?st/im`) - - So(err, ShouldBeNil) - So(p, ShouldNotBeNil) - So(p.Expression, ShouldEqual, "te?st") - So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) - - So(p.String(), ShouldEqual, "/te?st/im") - }) - }) - - Convey("When parse literal with id and flags", func() { - p, err := hyperscan.ParseLiteral("3:/foobar/iu") - - So(err, ShouldBeNil) - So(p.Id, ShouldEqual, 3) - So(p.Expression, ShouldEqual, "foobar") - So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.Utf8Mode) - }) - - Convey("When parse with a lot of flags", func() { - p, err := hyperscan.ParseLiteral(`/test/ismoeupf`) - - So(err, ShouldBeNil) - So(p, ShouldNotBeNil) - So(p.Expression, ShouldEqual, "test") - So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.DotAll|hyperscan.MultiLine|hyperscan.SingleMatch| - hyperscan.AllowEmpty|hyperscan.Utf8Mode|hyperscan.UnicodeProperty|hyperscan.PrefilterMode) - - So(p.Flags.String(), ShouldEqual, "8HPVWims") - So(p.String(), ShouldEqual, "/test/8HPVWims") - }) - - Convey("When parse without flags", func() { - p, err := hyperscan.ParseLiteral(`test`) - - So(err, ShouldBeNil) - So(p, ShouldNotBeNil) - So(p.Expression, ShouldEqual, "test") - So(p.Flags, ShouldEqual, 0) - - Convey("When literal contains regular grammar", func() { - p, err := hyperscan.ParseLiteral(`/te?st/im`) - - So(err, ShouldBeNil) - So(p, ShouldNotBeNil) - So(p.Expression, ShouldEqual, "te?st") - So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) - - So(p.String(), ShouldEqual, "/te?st/im") - }) - }) - - Convey("When literal is valid", func() { - p := hyperscan.NewLiteral("test") - - info, err := p.Info() - - So(err, ShouldBeNil) - So(info, ShouldNotBeNil) - So(info, ShouldResemble, &hyperscan.ExprInfo{MinWidth: 4, MaxWidth: 4}) - So(p.IsValid(), ShouldBeTrue) - }) - - Convey("When quote a string", func() { - So(hyperscan.Quote("test"), ShouldEqual, "`test`") - So(hyperscan.Quote("`can't backquote this`"), ShouldEqual, "\"`can't backquote this`\"") - }) - }) -} - func TestDatabaseBuilderV5(t *testing.T) { Convey("Given a DatabaseBuilder (v5)", t, func() { b := hyperscan.DatabaseBuilder{} diff --git a/hyperscan/doc.go b/hyperscan/doc.go index 2540d7b..9206a79 100644 --- a/hyperscan/doc.go +++ b/hyperscan/doc.go @@ -1,7 +1,7 @@ // Package hyperscan is the Golang binding for Intel's HyperScan regex matching library: // [hyperscan.io](https://www.hyperscan.io/) // -// Hyperscan (https://github.com/01org/hyperscan) is a software regular expression matching engine +// Hyperscan (https://github.com/intel/hyperscan) is a software regular expression matching engine // designed with high performance and flexibility in mind. // It is implemented as a library that exposes a straightforward C API. // @@ -10,7 +10,7 @@ // Hyperscan is typically used in a DPI library stack. // The Hyperscan API itself is composed of two major components: // -// ## Compilation +// Compilation // // These functions take a group of regular expressions, along with identifiers and option flags, // and compile them into an immutable database that can be used by the Hyperscan scanning API. @@ -21,9 +21,10 @@ // Compiled databases can be serialized and relocated, so that they can be stored to disk or moved between hosts. // They can also be targeted to particular platform features // (for example, the use of Intel® Advanced Vector Extensions 2 (Intel® AVX2) instructions). -// See Compiling Patterns for more detail. (http://01org.github.io/hyperscan/dev-reference/compilation.html) // -// ## Scanning +// See Compiling Patterns for more detail. (http://intel.github.io/hyperscan/dev-reference/compilation.html) +// +// Scanning // // Once a Hyperscan database has been created, it can be used to scan data in memory. // Hyperscan provides several scanning modes, depending on whether the data to be scanned is available @@ -35,22 +36,26 @@ // // 1. No memory allocations occur at runtime with the exception of two fixed-size allocations, // both of which should be done ahead of time for performance-critical applications: -// - Scratch space: temporary memory used for internal data at scan time. -// Structures in scratch space do not persist beyond the end of a single scan call. -// - Stream state: in streaming mode only, some state space is required to store -// data that persists between scan calls for each stream. This allows Hyperscan to -// track matches that span multiple blocks of data. +// +// 1.1 Scratch space: temporary memory used for internal data at scan time. +// Structures in scratch space do not persist beyond the end of a single scan call. +// +// 1.2 Stream state: in streaming mode only, some state space is required to store +// data that persists between scan calls for each stream. This allows Hyperscan to +// track matches that span multiple blocks of data. +// // 2. The sizes of the scratch space and stream state (in streaming mode) required for a given database are fixed // and determined at database compile time. // This means that the memory requirements of the application are known ahead of time, and these structures // can be pre-allocated if required for performance reasons. +// // 3. Any pattern that has successfully been compiled by the Hyperscan compiler can be scanned against any input. // There are no internal resource limits or other limitations at runtime // that could cause a scan call to return an error. // -// See Scanning for Patterns for more detail. (http://01org.github.io/hyperscan/dev-reference/runtime.html) +// See Scanning for Patterns for more detail. (http://intel.github.io/hyperscan/dev-reference/runtime.html) // -// ## Building a Database +// Building a Database // // The Hyperscan compiler API accepts regular expressions and converts them into a compiled pattern database // that can then be used to scan data. @@ -59,12 +64,14 @@ // When compiling expressions, a decision needs to be made whether the resulting compiled patterns are to be used // in a streaming, block or vectored mode: // -// - Streaming mode: the target data to be scanned is a continuous stream, not all of -// which is available at once; blocks of data are scanned in sequence and matches may -// span multiple blocks in a stream. In streaming mode, each stream requires a block -// of memory to store its state between scan calls. -// - Block mode: the target data is a discrete, contiguous block which can be scanned -// in one call and does not require state to be retained. -// - Vectored mode: the target data consists of a list of non-contiguous blocks that are -// available all at once. As for block mode, no retention of state is required. +// 1. Streaming mode: the target data to be scanned is a continuous stream, not all of +// which is available at once; blocks of data are scanned in sequence and matches may +// span multiple blocks in a stream. In streaming mode, each stream requires a block +// of memory to store its state between scan calls. +// +// 2. Block mode: the target data is a discrete, contiguous block which can be scanned +// in one call and does not require state to be retained. +// +// 3. Vectored mode: the target data consists of a list of non-contiguous blocks that are +// available all at once. As for block mode, no retention of state is required. package hyperscan diff --git a/hyperscan/example_api_test.go b/hyperscan/example_api_test.go index 5d3a269..bbfa7a6 100644 --- a/hyperscan/example_api_test.go +++ b/hyperscan/example_api_test.go @@ -17,7 +17,7 @@ func ExampleMatch() { // Output: // true // false - // false parse pattern, invalid pattern `a(b`, Missing close parenthesis for group started at index 1. + // false parse pattern, pattern `a(b`, Missing close parenthesis for group started at index 1. } func ExampleMatchReader() { @@ -31,5 +31,5 @@ func ExampleMatchReader() { // Output: // true // false - // false parse pattern, invalid pattern `a(b`, Missing close parenthesis for group started at index 1. + // false parse pattern, pattern `a(b`, Missing close parenthesis for group started at index 1. } diff --git a/hyperscan/internal.go b/hyperscan/internal.go deleted file mode 100644 index abbbc2f..0000000 --- a/hyperscan/internal.go +++ /dev/null @@ -1,1253 +0,0 @@ -package hyperscan - -import ( - "errors" - "fmt" - "reflect" - "runtime" - "sort" - "strconv" - "strings" - "unsafe" - - "github.com/flier/gohs/hyperscan/handle" -) - -/* -#cgo pkg-config: libhs -#cgo linux LDFLAGS: -lm - -#include -#include -#include - -#include - -static inline void* aligned64_malloc(size_t size) { - void* result; -#ifdef _WIN32 - result = _aligned_malloc(size, 64); -#else - if (posix_memalign(&result, 64, size)) { - result = 0; - } -#endif - return result; -} - -static inline void aligned64_free(void *ptr) { -#ifdef _WIN32 - _aligned_free(ptr); -#else - free(ptr); -#endif -} - -#define DEFINE_ALLOCTOR(ID, TYPE) \ - extern void *hs ## ID ## Alloc(size_t size); \ - extern void hs ## ID ## Free(void *ptr); \ - static inline void *hs ## ID ## Alloc_cgo(size_t size) { return hs ## ID ## Alloc(size); } \ - static inline void hs ## ID ## Free_cgo(void *ptr) { hs ## ID ## Free(ptr); } \ - static inline hs_error_t hs_set_ ## TYPE ## _allocator_cgo() \ - { return hs_set_ ## TYPE ## _allocator(hs ## ID ## Alloc_cgo, hs ## ID ## Free_cgo); } \ - static inline hs_error_t hs_clear_ ## TYPE ## _allocator_cgo() \ - { return hs_set_ ## TYPE ## _allocator(NULL, NULL); } - -DEFINE_ALLOCTOR(Db, database); -DEFINE_ALLOCTOR(Misc, misc); -DEFINE_ALLOCTOR(Scratch, scratch); -DEFINE_ALLOCTOR(Stream, stream); - -extern int hsMatchEventCallback(unsigned int id, unsigned long long from, unsigned long long to, unsigned int flags, void *context); -*/ -import "C" - -// CompileFlag represents a pattern flag -type CompileFlag uint - -const ( - // Caseless represents set case-insensitive matching. - Caseless CompileFlag = C.HS_FLAG_CASELESS - // DotAll represents matching a `.` will not exclude newlines. - DotAll CompileFlag = C.HS_FLAG_DOTALL - // MultiLine set multi-line anchoring. - MultiLine CompileFlag = C.HS_FLAG_MULTILINE - // SingleMatch set single-match only mode. - SingleMatch CompileFlag = C.HS_FLAG_SINGLEMATCH - // AllowEmpty allow expressions that can match against empty buffers. - AllowEmpty CompileFlag = C.HS_FLAG_ALLOWEMPTY - // Utf8Mode enable UTF-8 mode for this expression. - Utf8Mode CompileFlag = C.HS_FLAG_UTF8 - // UnicodeProperty enable Unicode property support for this expression. - UnicodeProperty CompileFlag = C.HS_FLAG_UCP - // PrefilterMode enable prefiltering mode for this expression. - PrefilterMode CompileFlag = C.HS_FLAG_PREFILTER - // SomLeftMost enable leftmost start of match reporting. - SomLeftMost CompileFlag = C.HS_FLAG_SOM_LEFTMOST -) - -var compileFlags = map[rune]CompileFlag{ - 'i': Caseless, - 's': DotAll, - 'm': MultiLine, - 'H': SingleMatch, - 'V': AllowEmpty, - '8': Utf8Mode, - 'W': UnicodeProperty, - 'P': PrefilterMode, - 'L': SomLeftMost, -} - -var deprecatedCompileFlags = map[rune]CompileFlag{ - 'o': SingleMatch, - 'e': AllowEmpty, - 'u': Utf8Mode, - 'p': UnicodeProperty, - 'f': PrefilterMode, - 'l': SomLeftMost, -} - -/* -ParseCompileFlag parse the compile pattern flags from string - - i Caseless Case-insensitive matching - s DotAll Dot (.) will match newlines - m MultiLine Multi-line anchoring - H SingleMatch Report match ID at most once (`o` deprecated) - V AllowEmpty Allow patterns that can match against empty buffers (`e` deprecated) - 8 Utf8Mode UTF-8 mode (`u` deprecated) - W UnicodeProperty Unicode property support (`p` deprecated) - P PrefilterMode Prefiltering mode (`f` deprecated) - L SomLeftMost Leftmost start of match reporting (`l` deprecated) - C Combination Logical combination of patterns (Hyperscan 5.0) - Q Quiet Quiet at matching (Hyperscan 5.0) -*/ -func ParseCompileFlag(s string) (CompileFlag, error) { - var flags CompileFlag - - for _, c := range s { - if flag, exists := compileFlags[c]; exists { - flags |= flag - } else if flag, exists := deprecatedCompileFlags[c]; exists { - flags |= flag - } else { - return 0, fmt.Errorf("flag `%c`, %w", c, ErrUnexpected) - } - } - - return flags, nil -} - -func (flags CompileFlag) String() string { - var values []string - - for c, flag := range compileFlags { - if (flags & flag) == flag { - values = append(values, string(c)) - } - } - - sort.Strings(values) - - return strings.Join(values, "") -} - -// CpuFeature is the CPU feature support flags -type CpuFeature int // nolint: golint,stylecheck - -const ( - // AVX2 is a CPU features flag indicates that the target platform supports AVX2 instructions. - AVX2 CpuFeature = C.HS_CPU_FEATURES_AVX2 - // AVX512 is a CPU features flag indicates that the target platform supports AVX512 instructions, specifically AVX-512BW. Using AVX512 implies the use of AVX2. - AVX512 CpuFeature = C.HS_CPU_FEATURES_AVX512 -) - -// TuneFlag is the tuning flags -type TuneFlag int - -const ( - // Generic indicates that the compiled database should not be tuned for any particular target platform. - Generic TuneFlag = C.HS_TUNE_FAMILY_GENERIC - // SandyBridge indicates that the compiled database should be tuned for the Sandy Bridge microarchitecture. - SandyBridge TuneFlag = C.HS_TUNE_FAMILY_SNB - // IvyBridge indicates that the compiled database should be tuned for the Ivy Bridge microarchitecture. - IvyBridge TuneFlag = C.HS_TUNE_FAMILY_IVB - // Haswell indicates that the compiled database should be tuned for the Haswell microarchitecture. - Haswell TuneFlag = C.HS_TUNE_FAMILY_HSW - // Silvermont indicates that the compiled database should be tuned for the Silvermont microarchitecture. - Silvermont TuneFlag = C.HS_TUNE_FAMILY_SLM - // Broadwell indicates that the compiled database should be tuned for the Broadwell microarchitecture. - Broadwell TuneFlag = C.HS_TUNE_FAMILY_BDW - // Skylake indicates that the compiled database should be tuned for the Skylake microarchitecture. - Skylake TuneFlag = C.HS_TUNE_FAMILY_SKL - // SkylakeServer indicates that the compiled database should be tuned for the Skylake Server microarchitecture. - SkylakeServer TuneFlag = C.HS_TUNE_FAMILY_SKX - // Goldmont indicates that the compiled database should be tuned for the Goldmont microarchitecture. - Goldmont TuneFlag = C.HS_TUNE_FAMILY_GLM -) - -// ModeFlag represents the compile mode flags -type ModeFlag uint - -const ( - // BlockMode for the block scan (non-streaming) database. - BlockMode ModeFlag = C.HS_MODE_BLOCK - // NoStreamMode is alias for Block. - NoStreamMode ModeFlag = C.HS_MODE_NOSTREAM - // StreamMode for the streaming database. - StreamMode ModeFlag = C.HS_MODE_STREAM - // VectoredMode for the vectored scanning database. - VectoredMode ModeFlag = C.HS_MODE_VECTORED - // SomHorizonLargeMode use full precision to track start of match offsets in stream state. - SomHorizonLargeMode ModeFlag = C.HS_MODE_SOM_HORIZON_LARGE - // SomHorizonMediumMode use medium precision to track start of match offsets in stream state. (within 2^32 bytes) - SomHorizonMediumMode ModeFlag = C.HS_MODE_SOM_HORIZON_MEDIUM - // SomHorizonSmallMode use limited precision to track start of match offsets in stream state. (within 2^16 bytes) - SomHorizonSmallMode ModeFlag = C.HS_MODE_SOM_HORIZON_SMALL - // ModeMask represents the mask of database mode - ModeMask ModeFlag = 0xFF -) - -var modeFlags = map[string]ModeFlag{ - "STREAM": StreamMode, - "NOSTREAM": BlockMode, - "VECTORED": VectoredMode, - "BLOCK": BlockMode, -} - -// ParseModeFlag parse a database mode from string -func ParseModeFlag(s string) (ModeFlag, error) { - if mode, exists := modeFlags[strings.ToUpper(s)]; exists { - return mode, nil - } - - return BlockMode, fmt.Errorf("database mode %s, %w", s, ErrUnexpected) -} - -func (m ModeFlag) String() string { - switch m & 0xF { - case BlockMode: - return "BLOCK" - case StreamMode: - return "STREAM" - case VectoredMode: - return "VECTORED" - default: - panic(fmt.Sprintf("unknown mode: %d", m)) - } -} - -// ScanFlag represents a scan flag -type ScanFlag uint - -// HsError represents an error -type HsError int - -const ( - // ErrSuccess is the error returned if the engine completed normally. - ErrSuccess HsError = C.HS_SUCCESS - // ErrInvalid is the error returned if a parameter passed to this function was invalid. - ErrInvalid HsError = C.HS_INVALID - // ErrNoMemory is the error returned if a memory allocation failed. - ErrNoMemory HsError = C.HS_NOMEM - // ErrScanTerminated is the error returned if the engine was terminated by callback. - ErrScanTerminated HsError = C.HS_SCAN_TERMINATED - // ErrCompileError is the error returned if the pattern compiler failed. - ErrCompileError HsError = C.HS_COMPILER_ERROR - // ErrDatabaseVersionError is the error returned if the given database was built for a different version of Hyperscan. - ErrDatabaseVersionError HsError = C.HS_DB_VERSION_ERROR - // ErrDatabasePlatformError is the error returned if the given database was built for a different platform (i.e., CPU type). - ErrDatabasePlatformError HsError = C.HS_DB_PLATFORM_ERROR - // ErrDatabaseModeError is the error returned if the given database was built for a different mode of operation. - ErrDatabaseModeError HsError = C.HS_DB_MODE_ERROR - // ErrBadAlign is the error returned if a parameter passed to this function was not correctly aligned. - ErrBadAlign HsError = C.HS_BAD_ALIGN - // ErrBadAlloc is the error returned if the memory allocator did not correctly return memory suitably aligned. - ErrBadAlloc HsError = C.HS_BAD_ALLOC - // ErrScratchInUse is the error returned if the scratch region was already in use. - ErrScratchInUse HsError = C.HS_SCRATCH_IN_USE - // ErrArchError is the error returned if unsupported CPU architecture. - ErrArchError HsError = C.HS_ARCH_ERROR - // ErrInsufficientSpace is the error returned if provided buffer was too small. - ErrInsufficientSpace HsError = C.HS_INSUFFICIENT_SPACE -) - -var hsErrorMessages = map[HsError]string{ - C.HS_SUCCESS: "The engine completed normally.", - C.HS_INVALID: "A parameter passed to this function was invalid.", - C.HS_NOMEM: "A memory allocation failed.", - C.HS_SCAN_TERMINATED: "The engine was terminated by callback.", - C.HS_COMPILER_ERROR: "The pattern compiler failed.", - C.HS_DB_VERSION_ERROR: "The given database was built for a different version of Hyperscan.", - C.HS_DB_PLATFORM_ERROR: "The given database was built for a different platform (i.e., CPU type).", - C.HS_DB_MODE_ERROR: "The given database was built for a different mode of operation.", - C.HS_BAD_ALIGN: "A parameter passed to this function was not correctly aligned.", - C.HS_BAD_ALLOC: "The memory allocator did not correctly return aligned memory.", - C.HS_SCRATCH_IN_USE: "The scratch region was already in use.", - C.HS_ARCH_ERROR: "Unsupported CPU architecture.", - C.HS_INSUFFICIENT_SPACE: "Provided buffer was too small.", -} - -func (e HsError) Error() string { - if msg, exists := hsErrorMessages[e]; exists { - return msg - } - - return fmt.Sprintf("unexpected error, %d", int(e)) -} - -type compileError struct { - msg string - expr int -} - -// A human-readable error message describing the error. -func (e *compileError) Error() string { return e.msg } - -// The zero-based number of the expression that caused the error (if this can be determined). -// If the error is not specific to an expression, then this value will be less than zero. -func (e *compileError) Expression() int { return e.expr } - -type hsPlatformInfo struct { - platform C.struct_hs_platform_info -} - -// Tune returns the tuning flags of the platform. -func (i *hsPlatformInfo) Tune() TuneFlag { return TuneFlag(i.platform.tune) } - -// CpuFeatures returns the CPU features of the platform. -func (i *hsPlatformInfo) CpuFeatures() CpuFeature { return CpuFeature(i.platform.cpu_features) } // nolint: golint,stylecheck - -func newPlatformInfo(tune TuneFlag, cpu CpuFeature) *hsPlatformInfo { - var platform C.struct_hs_platform_info - - platform.tune = C.uint(tune) - platform.cpu_features = C.ulonglong(cpu) - - return &hsPlatformInfo{platform} -} - -func hsPopulatePlatform() (*hsPlatformInfo, error) { - var platform C.struct_hs_platform_info - - if ret := C.hs_populate_platform(&platform); ret != C.HS_SUCCESS { - return nil, HsError(ret) - } - - return &hsPlatformInfo{platform}, nil -} - -type ( - hsDatabase *C.hs_database_t - hsScratch *C.hs_scratch_t - hsStream *C.hs_stream_t -) - -// ExprInfo containing information related to an expression -type ExprInfo struct { - MinWidth uint // The minimum length in bytes of a match for the pattern. - MaxWidth uint // The maximum length in bytes of a match for the pattern. - ReturnUnordered bool // Whether this expression can produce matches that are not returned in order, such as those produced by assertions. - AtEndOfData bool // Whether this expression can produce matches at end of data (EOD). - OnlyAtEndOfData bool // Whether this expression can *only* produce matches at end of data (EOD). -} - -// UnboundedMaxWidth represents the pattern expression has an unbounded maximum width -const UnboundedMaxWidth = C.UINT_MAX - -func newExprInfo(info *C.hs_expr_info_t) *ExprInfo { - return &ExprInfo{ - MinWidth: uint(info.min_width), - MaxWidth: uint(info.max_width), - ReturnUnordered: info.unordered_matches != 0, - AtEndOfData: info.matches_at_eod != 0, - OnlyAtEndOfData: info.matches_only_at_eod != 0, - } -} - -// ExtFlag are used in ExprExt.Flags to indicate which fields are used. -type ExtFlag uint64 - -const ( - // ExtMinOffset is a flag indicating that the ExprExt.MinOffset field is used. - ExtMinOffset ExtFlag = C.HS_EXT_FLAG_MIN_OFFSET - // ExtMaxOffset is a flag indicating that the ExprExt.MaxOffset field is used. - ExtMaxOffset ExtFlag = C.HS_EXT_FLAG_MAX_OFFSET - // ExtMinLength is a flag indicating that the ExprExt.MinLength field is used. - ExtMinLength ExtFlag = C.HS_EXT_FLAG_MIN_LENGTH - // ExtEditDistance is a flag indicating that the ExprExt.EditDistance field is used. - ExtEditDistance ExtFlag = C.HS_EXT_FLAG_EDIT_DISTANCE - // ExtHammingDistance is a flag indicating that the ExprExt.HammingDistance field is used. - ExtHammingDistance ExtFlag = C.HS_EXT_FLAG_HAMMING_DISTANCE -) - -// Ext is a option containing additional parameters related to an expression. -type Ext func(ext *ExprExt) - -// MinOffset given the minimum end offset in the data stream at which this expression should match successfully. -func MinOffset(n uint64) Ext { - return func(ext *ExprExt) { - ext.Flags |= ExtMinOffset - ext.MinOffset = n - } -} - -// MaxOffset given the maximum end offset in the data stream at which this expression should match successfully. -func MaxOffset(n uint64) Ext { - return func(ext *ExprExt) { - ext.Flags |= ExtMaxOffset - ext.MaxOffset = n - } -} - -// MinLength given the minimum match length (from start to end) required to successfully match this expression. -func MinLength(n uint64) Ext { - return func(ext *ExprExt) { - ext.Flags |= ExtMinLength - ext.MinLength = n - } -} - -// EditDistance allow patterns to approximately match within this edit distance. -func EditDistance(n uint32) Ext { - return func(ext *ExprExt) { - ext.Flags |= ExtEditDistance - ext.EditDistance = n - } -} - -// HammingDistance allow patterns to approximately match within this Hamming distance. -func HammingDistance(n uint32) Ext { - return func(ext *ExprExt) { - ext.Flags |= ExtHammingDistance - ext.HammingDistance = n - } -} - -// ExprExt is a structure containing additional parameters related to an expression. -type ExprExt struct { - Flags ExtFlag // Flags governing which parts of this structure are to be used by the compiler. - MinOffset uint64 // The minimum end offset in the data stream at which this expression should match successfully. - MaxOffset uint64 // The maximum end offset in the data stream at which this expression should match successfully. - MinLength uint64 // The minimum match length (from start to end) required to successfully match this expression. - EditDistance uint32 // Allow patterns to approximately match within this edit distance. - HammingDistance uint32 // Allow patterns to approximately match within this Hamming distance. -} - -// With specifies the additional parameters related to an expression. -func (ext *ExprExt) With(exts ...Ext) *ExprExt { - for _, f := range exts { - f(ext) - } - - return ext -} - -func (ext *ExprExt) String() string { - var values []string - - if (ext.Flags & ExtMinOffset) == ExtMinOffset { - values = append(values, fmt.Sprintf("min_offset=%d", ext.MinOffset)) - } - - if (ext.Flags & ExtMaxOffset) == ExtMaxOffset { - values = append(values, fmt.Sprintf("max_offset=%d", ext.MaxOffset)) - } - - if (ext.Flags & ExtMinLength) == ExtMinLength { - values = append(values, fmt.Sprintf("min_length=%d", ext.MinLength)) - } - - if (ext.Flags & ExtEditDistance) == ExtEditDistance { - values = append(values, fmt.Sprintf("edit_distance=%d", ext.EditDistance)) - } - - if (ext.Flags & ExtHammingDistance) == ExtHammingDistance { - values = append(values, fmt.Sprintf("hamming_distance=%d", ext.HammingDistance)) - } - - return "{" + strings.Join(values, ",") + "}" -} - -const keyValuePair = 2 - -// ParseExprExt parse containing additional parameters from string -func ParseExprExt(s string) (ext *ExprExt, err error) { - ext = new(ExprExt) - - if strings.HasPrefix(s, "{") && strings.HasSuffix(s, "}") { - s = strings.TrimSuffix(strings.TrimPrefix(s, "{"), "}") - } - - for _, s := range strings.Split(s, ",") { - parts := strings.SplitN(s, "=", keyValuePair) - - if len(parts) != keyValuePair { - continue - } - - key := strings.ToLower(parts[0]) - value := parts[1] - - var n int - - if n, err = strconv.Atoi(value); err != nil { - return - } - - switch key { - case "min_offset": - ext.Flags |= ExtMinOffset - ext.MinOffset = uint64(n) - - case "max_offset": - ext.Flags |= ExtMaxOffset - ext.MaxOffset = uint64(n) - - case "min_length": - ext.Flags |= ExtMinLength - ext.MinLength = uint64(n) - - case "edit_distance": - ext.Flags |= ExtEditDistance - ext.EditDistance = uint32(n) - - case "hamming_distance": - ext.Flags |= ExtHammingDistance - ext.HammingDistance = uint32(n) - } - } - - return // nolint: nakedret -} - -type ( - hsAllocFunc func(uint) unsafe.Pointer - hsFreeFunc func(unsafe.Pointer) -) - -type hsAllocator struct { - Alloc hsAllocFunc - Free hsFreeFunc -} - -var ( - dbAllocator hsAllocator - miscAllocator hsAllocator - scratchAllocator hsAllocator - streamAllocator hsAllocator -) - -func hsDefaultAlloc(size uint) unsafe.Pointer { - return C.aligned64_malloc(C.size_t(size)) -} - -func hsDefaultFree(ptr unsafe.Pointer) { - C.aligned64_free(ptr) -} - -//export hsDbAlloc -func hsDbAlloc(size C.size_t) unsafe.Pointer { - if dbAllocator.Alloc != nil { - return dbAllocator.Alloc(uint(size)) - } - - return hsDefaultAlloc(uint(size)) -} - -//export hsDbFree -func hsDbFree(ptr unsafe.Pointer) { - if dbAllocator.Free != nil { - dbAllocator.Free(ptr) - } else { - hsDefaultFree(ptr) - } -} - -func hsSetDatabaseAllocator(allocFunc hsAllocFunc, freeFunc hsFreeFunc) error { - dbAllocator = hsAllocator{allocFunc, freeFunc} - - if ret := C.hs_set_database_allocator_cgo(); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsClearDatabaseAllocator() error { - if ret := C.hs_clear_database_allocator_cgo(); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -//export hsMiscAlloc -func hsMiscAlloc(size C.size_t) unsafe.Pointer { - if miscAllocator.Alloc != nil { - return miscAllocator.Alloc(uint(size)) - } - - return hsDefaultAlloc(uint(size)) -} - -//export hsMiscFree -func hsMiscFree(ptr unsafe.Pointer) { - if miscAllocator.Free != nil { - miscAllocator.Free(ptr) - } else { - hsDefaultFree(ptr) - } -} - -func hsSetMiscAllocator(allocFunc hsAllocFunc, freeFunc hsFreeFunc) error { - miscAllocator = hsAllocator{allocFunc, freeFunc} - - if ret := C.hs_set_misc_allocator_cgo(); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsClearMiscAllocator() error { - if ret := C.hs_clear_misc_allocator_cgo(); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -//export hsScratchAlloc -func hsScratchAlloc(size C.size_t) unsafe.Pointer { - if scratchAllocator.Alloc != nil { - return scratchAllocator.Alloc(uint(size)) - } - - return hsDefaultAlloc(uint(size)) -} - -//export hsScratchFree -func hsScratchFree(ptr unsafe.Pointer) { - if scratchAllocator.Free != nil { - scratchAllocator.Free(ptr) - } else { - hsDefaultFree(ptr) - } -} - -func hsSetScratchAllocator(allocFunc hsAllocFunc, freeFunc hsFreeFunc) error { - scratchAllocator = hsAllocator{allocFunc, freeFunc} - - if ret := C.hs_set_scratch_allocator_cgo(); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsClearScratchAllocator() error { - if ret := C.hs_clear_scratch_allocator_cgo(); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -//export hsStreamAlloc -func hsStreamAlloc(size C.size_t) unsafe.Pointer { - if streamAllocator.Alloc != nil { - return streamAllocator.Alloc(uint(size)) - } - - return hsDefaultAlloc(uint(size)) -} - -//export hsStreamFree -func hsStreamFree(ptr unsafe.Pointer) { - if streamAllocator.Free != nil { - streamAllocator.Free(ptr) - } else { - hsDefaultFree(ptr) - } -} - -func hsSetStreamAllocator(allocFunc hsAllocFunc, freeFunc hsFreeFunc) error { - streamAllocator = hsAllocator{allocFunc, freeFunc} - - if ret := C.hs_set_stream_allocator_cgo(); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsClearStreamAllocator() error { - if ret := C.hs_clear_stream_allocator_cgo(); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsVersion() string { - return C.GoString(C.hs_version()) -} - -func hsValidPlatform() error { - if ret := C.hs_valid_platform(); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsFreeDatabase(db hsDatabase) (err error) { - if ret := C.hs_free_database(db); ret != C.HS_SUCCESS { - err = HsError(ret) - } - - return -} - -func hsSerializeDatabase(db hsDatabase) (b []byte, err error) { - var data *C.char - var length C.size_t - - ret := C.hs_serialize_database(db, &data, &length) - if ret != C.HS_SUCCESS { - err = HsError(ret) - } else { - defer C.free(unsafe.Pointer(data)) - - b = C.GoBytes(unsafe.Pointer(data), C.int(length)) - } - - return -} - -func hsDeserializeDatabase(data []byte) (hsDatabase, error) { - var db *C.hs_database_t - - ret := C.hs_deserialize_database((*C.char)(unsafe.Pointer(&data[0])), C.size_t(len(data)), &db) - - runtime.KeepAlive(data) - - if ret != C.HS_SUCCESS { - return nil, HsError(ret) - } - - return db, nil -} - -func hsDeserializeDatabaseAt(data []byte, db hsDatabase) error { - ret := C.hs_deserialize_database_at((*C.char)(unsafe.Pointer(&data[0])), C.size_t(len(data)), db) - - runtime.KeepAlive(data) - - if ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsStreamSize(db hsDatabase) (int, error) { - var size C.size_t - - if ret := C.hs_stream_size(db, &size); ret != C.HS_SUCCESS { - return 0, HsError(ret) - } - - return int(size), nil -} - -func hsDatabaseSize(db hsDatabase) (int, error) { - var size C.size_t - - if ret := C.hs_database_size(db, &size); ret != C.HS_SUCCESS { - return -1, HsError(ret) - } - - return int(size), nil -} - -func hsSerializedDatabaseSize(data []byte) (int, error) { - var size C.size_t - - ret := C.hs_serialized_database_size((*C.char)(unsafe.Pointer(&data[0])), C.size_t(len(data)), &size) - - runtime.KeepAlive(data) - - if ret != C.HS_SUCCESS { - return 0, HsError(ret) - } - - return int(size), nil -} - -func hsDatabaseInfo(db hsDatabase) (string, error) { - var info *C.char - - if ret := C.hs_database_info(db, &info); ret != C.HS_SUCCESS { - return "", HsError(ret) - } - - defer C.free(unsafe.Pointer(info)) - - return C.GoString(info), nil -} - -func hsSerializedDatabaseInfo(data []byte) (string, error) { - var info *C.char - - ret := C.hs_serialized_database_info((*C.char)(unsafe.Pointer(&data[0])), C.size_t(len(data)), &info) - - runtime.KeepAlive(data) - - if ret != C.HS_SUCCESS { - return "", HsError(ret) - } - - defer C.free(unsafe.Pointer(info)) - - return C.GoString(info), nil -} - -func hsCompile(expression string, flags CompileFlag, mode ModeFlag, info *hsPlatformInfo) (hsDatabase, error) { - var db *C.hs_database_t - var err *C.hs_compile_error_t - var platform *C.hs_platform_info_t - - if info != nil { - platform = &info.platform - } - - expr := C.CString(expression) - - defer C.free(unsafe.Pointer(expr)) - - ret := C.hs_compile(expr, C.uint(flags), C.uint(mode), platform, &db, &err) - - if err != nil { - defer C.hs_free_compile_error(err) - } - - if ret == C.HS_SUCCESS { - return db, nil - } - - if ret == C.HS_COMPILER_ERROR && err != nil { - return nil, &compileError{C.GoString(err.message), int(err.expression)} - } - - return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) -} - -func hsCompileMulti(patterns []*Pattern, mode ModeFlag, info *hsPlatformInfo) (hsDatabase, error) { - var db *C.hs_database_t - var err *C.hs_compile_error_t - var platform *C.hs_platform_info_t - - if info != nil { - platform = &info.platform - } - - cexprs := (**C.char)(C.calloc(C.size_t(len(patterns)), C.size_t(unsafe.Sizeof(uintptr(0))))) - exprs := (*[1 << 30]*C.char)(unsafe.Pointer(cexprs))[:len(patterns):len(patterns)] - - cflags := (*C.uint)(C.calloc(C.size_t(len(patterns)), C.size_t(unsafe.Sizeof(C.uint(0))))) - flags := (*[1 << 30]C.uint)(unsafe.Pointer(cflags))[:len(patterns):len(patterns)] - - cids := (*C.uint)(C.calloc(C.size_t(len(patterns)), C.size_t(unsafe.Sizeof(C.uint(0))))) - ids := (*[1 << 30]C.uint)(unsafe.Pointer(cids))[:len(patterns):len(patterns)] - - cexts := (**C.hs_expr_ext_t)(C.calloc(C.size_t(len(patterns)), C.size_t(unsafe.Sizeof(uintptr(0))))) - exts := (*[1 << 30]*C.hs_expr_ext_t)(unsafe.Pointer(cexts))[:len(patterns):len(patterns)] - - for i, pattern := range patterns { - exprs[i] = C.CString(string(pattern.Expression)) - flags[i] = C.uint(pattern.Flags) - ids[i] = C.uint(pattern.Id) - exts[i] = (*C.hs_expr_ext_t)(unsafe.Pointer(pattern.ext)) - } - - ret := C.hs_compile_ext_multi(cexprs, cflags, cids, cexts, C.uint(len(patterns)), C.uint(mode), platform, &db, &err) - - for _, expr := range exprs { - C.free(unsafe.Pointer(expr)) - } - - C.free(unsafe.Pointer(cexprs)) - C.free(unsafe.Pointer(cflags)) - C.free(unsafe.Pointer(cexts)) - C.free(unsafe.Pointer(cids)) - - runtime.KeepAlive(patterns) - - if err != nil { - defer C.hs_free_compile_error(err) - } - - if ret == C.HS_SUCCESS { - return db, nil - } - - if ret == C.HS_COMPILER_ERROR && err != nil { - return nil, &compileError{C.GoString(err.message), int(err.expression)} - } - - return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) -} - -func hsExpressionInfo(expression string, flags CompileFlag) (*ExprInfo, error) { - var info *C.hs_expr_info_t - var err *C.hs_compile_error_t - - expr := C.CString(expression) - - defer C.free(unsafe.Pointer(expr)) - - ret := C.hs_expression_info(expr, C.uint(flags), &info, &err) - - if ret == C.HS_SUCCESS && info != nil { - defer hsMiscFree(unsafe.Pointer(info)) - - return newExprInfo(info), nil - } - - if err != nil { - defer C.hs_free_compile_error(err) - } - - if ret == C.HS_COMPILER_ERROR && err != nil { - return nil, &compileError{C.GoString(err.message), int(err.expression)} - } - - return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) -} - -func hsExpressionExt(expression string, flags CompileFlag) (ext *ExprExt, info *ExprInfo, err error) { - var exprInfo *C.hs_expr_info_t - var compileErr *C.hs_compile_error_t - - ext = new(ExprExt) - expr := C.CString(expression) - - defer C.free(unsafe.Pointer(expr)) - - ret := C.hs_expression_ext_info(expr, C.uint(flags), (*C.hs_expr_ext_t)(unsafe.Pointer(ext)), &exprInfo, &compileErr) - - if exprInfo != nil { - defer hsMiscFree(unsafe.Pointer(exprInfo)) - - info = newExprInfo(exprInfo) - } - - if compileErr != nil { - defer C.hs_free_compile_error(compileErr) - } - - if ret == C.HS_COMPILER_ERROR && compileErr != nil { - err = &compileError{C.GoString(compileErr.message), int(compileErr.expression)} - } else { - err = fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) - } - - return -} - -func hsAllocScratch(db hsDatabase) (hsScratch, error) { - var scratch *C.hs_scratch_t - - if ret := C.hs_alloc_scratch(db, &scratch); ret != C.HS_SUCCESS { - return nil, HsError(ret) - } - - return scratch, nil -} - -func hsReallocScratch(db hsDatabase, scratch *hsScratch) error { - if ret := C.hs_alloc_scratch(db, (**C.struct_hs_scratch)(scratch)); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsCloneScratch(scratch hsScratch) (hsScratch, error) { - var clone *C.hs_scratch_t - - if ret := C.hs_clone_scratch(scratch, &clone); ret != C.HS_SUCCESS { - return nil, HsError(ret) - } - - return clone, nil -} - -func hsScratchSize(scratch hsScratch) (int, error) { - var size C.size_t - - if ret := C.hs_scratch_size(scratch, &size); ret != C.HS_SUCCESS { - return 0, HsError(ret) - } - - return int(size), nil -} - -func hsFreeScratch(scratch hsScratch) error { - if ret := C.hs_free_scratch(scratch); ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -type hsMatchEventHandler func(id uint, from, to uint64, flags uint, context interface{}) error - -type hsMatchEventContext struct { - handler hsMatchEventHandler - context interface{} -} - -//export hsMatchEventCallback -func hsMatchEventCallback(id C.uint, from, to C.ulonglong, flags C.uint, data unsafe.Pointer) C.int { - ctx, ok := handle.Handle(data).Value().(hsMatchEventContext) - if !ok { - return C.HS_INVALID - } - - err := ctx.handler(uint(id), uint64(from), uint64(to), uint(flags), ctx.context) - if err != nil { - var hsErr HsError - if errors.As(err, &hsErr) { - return C.int(hsErr) - } - - return C.HS_SCAN_TERMINATED - } - - return C.HS_SUCCESS -} - -func hsScan(db hsDatabase, data []byte, flags ScanFlag, scratch hsScratch, onEvent hsMatchEventHandler, context interface{}) error { - if data == nil { - return HsError(C.HS_INVALID) - } - - h := handle.New(hsMatchEventContext{onEvent, context}) - defer h.Delete() - - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&data)) // FIXME: Zero-copy access to go data - - ret := C.hs_scan(db, - (*C.char)(unsafe.Pointer(hdr.Data)), - C.uint(hdr.Len), - C.uint(flags), - scratch, - C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) - - // Ensure go data is alive before the C function returns - runtime.KeepAlive(data) - - if ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsScanVector(db hsDatabase, data [][]byte, flags ScanFlag, scratch hsScratch, onEvent hsMatchEventHandler, context interface{}) error { - if data == nil { - return HsError(C.HS_INVALID) - } - - cdata := make([]uintptr, len(data)) - clength := make([]C.uint, len(data)) - - for i, d := range data { - if d == nil { - return HsError(C.HS_INVALID) - } - - // FIXME: Zero-copy access to go data - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&d)) // nolint: scopelint - cdata[i] = uintptr(unsafe.Pointer(hdr.Data)) - clength[i] = C.uint(hdr.Len) - } - - h := handle.New(hsMatchEventContext{onEvent, context}) - defer h.Delete() - - cdataHdr := (*reflect.SliceHeader)(unsafe.Pointer(&cdata)) // FIXME: Zero-copy access to go data - clengthHdr := (*reflect.SliceHeader)(unsafe.Pointer(&clength)) // FIXME: Zero-copy access to go data - - ret := C.hs_scan_vector(db, - (**C.char)(unsafe.Pointer(cdataHdr.Data)), - (*C.uint)(unsafe.Pointer(clengthHdr.Data)), - C.uint(cdataHdr.Len), - C.uint(flags), - scratch, - C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) - - // Ensure go data is alive before the C function returns - runtime.KeepAlive(data) - runtime.KeepAlive(cdata) - runtime.KeepAlive(clength) - - if ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsOpenStream(db hsDatabase, flags ScanFlag) (hsStream, error) { - var stream *C.hs_stream_t - - if ret := C.hs_open_stream(db, C.uint(flags), &stream); ret != C.HS_SUCCESS { - return nil, HsError(ret) - } - - return stream, nil -} - -func hsScanStream(stream hsStream, data []byte, flags ScanFlag, scratch hsScratch, onEvent hsMatchEventHandler, context interface{}) error { - if data == nil { - return HsError(C.HS_INVALID) - } - - h := handle.New(hsMatchEventContext{onEvent, context}) - defer h.Delete() - - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&data)) // FIXME: Zero-copy access to go data - - ret := C.hs_scan_stream(stream, - (*C.char)(unsafe.Pointer(hdr.Data)), - C.uint(hdr.Len), - C.uint(flags), - scratch, - C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) - - // Ensure go data is alive before the C function returns - runtime.KeepAlive(data) - - if ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsCloseStream(stream hsStream, scratch hsScratch, onEvent hsMatchEventHandler, context interface{}) error { - h := handle.New(hsMatchEventContext{onEvent, context}) - defer h.Delete() - - ret := C.hs_close_stream(stream, - scratch, - C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) - - if ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsResetStream(stream hsStream, flags ScanFlag, scratch hsScratch, onEvent hsMatchEventHandler, context interface{}) error { - h := handle.New(hsMatchEventContext{onEvent, context}) - defer h.Delete() - - ret := C.hs_reset_stream(stream, - C.uint(flags), - scratch, - C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) - - if ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsCopyStream(stream hsStream) (hsStream, error) { - var copied *C.hs_stream_t - - if ret := C.hs_copy_stream(&copied, stream); ret != C.HS_SUCCESS { - return nil, HsError(ret) - } - - return copied, nil -} - -func hsResetAndCopyStream(to, from hsStream, scratch hsScratch, onEvent hsMatchEventHandler, context interface{}) error { - h := handle.New(hsMatchEventContext{onEvent, context}) - defer h.Delete() - - ret := C.hs_reset_and_copy_stream(to, - from, - scratch, - C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) - - if ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsCompressStream(stream hsStream, buf []byte) ([]byte, error) { - var size C.size_t - - ret := C.hs_compress_stream(stream, (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)), &size) - - if ret == C.HS_INSUFFICIENT_SPACE { - buf = make([]byte, size) - - ret = C.hs_compress_stream(stream, (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)), &size) - } - - if ret != C.HS_SUCCESS { - return nil, HsError(ret) - } - - return buf[:size], nil -} - -func hsExpandStream(db hsDatabase, stream *hsStream, buf []byte) error { - ret := C.hs_expand_stream(db, (**C.hs_stream_t)(stream), (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf))) - - runtime.KeepAlive(buf) - - if ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} - -func hsResetAndExpandStream(stream hsStream, buf []byte, scratch hsScratch, onEvent hsMatchEventHandler, context interface{}) error { - h := handle.New(hsMatchEventContext{onEvent, context}) - defer h.Delete() - - ret := C.hs_reset_and_expand_stream(stream, - (*C.char)(unsafe.Pointer(&buf[0])), - C.size_t(len(buf)), - scratch, - C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) - - runtime.KeepAlive(buf) - - if ret != C.HS_SUCCESS { - return HsError(ret) - } - - return nil -} diff --git a/hyperscan/internal_test.go b/hyperscan/internal_test.go deleted file mode 100644 index ed27f5d..0000000 --- a/hyperscan/internal_test.go +++ /dev/null @@ -1,674 +0,0 @@ -//nolint:funlen -package hyperscan - -import ( - "regexp" - "testing" - "unsafe" - - . "github.com/smartystreets/goconvey/convey" -) - -func TestVersion(t *testing.T) { - Convey("Given a HyperScan version", t, func() { - ver := hsVersion() - - So(ver, ShouldNotBeEmpty) - - matched, err := regexp.MatchString(`^\d\.\d\.\d.*`, ver) - - So(err, ShouldBeNil) - So(matched, ShouldBeTrue) - }) -} - -func TestModeFlag(t *testing.T) { - Convey("Give a mode", t, func() { - So(BlockMode.String(), ShouldEqual, "BLOCK") - So(StreamMode.String(), ShouldEqual, "STREAM") - So(VectoredMode.String(), ShouldEqual, "VECTORED") - - Convey("When combile mode with flags", func() { - mode := StreamMode | SomHorizonLargeMode - - So(mode.String(), ShouldEqual, "STREAM") - }) - - Convey("When parse unknown mode", func() { - m, err := ParseModeFlag("test") - - So(err, ShouldNotBeNil) - So(err.Error(), ShouldContainSubstring, "database mode test") - So(m, ShouldEqual, BlockMode) - }) - }) -} - -func TestCompileFlag(t *testing.T) { - Convey("Given a compile flags", t, func() { - flags := Caseless | DotAll | MultiLine | SingleMatch | AllowEmpty | Utf8Mode | UnicodeProperty | PrefilterMode - - So(flags.String(), ShouldEqual, "8HPVWims") - - Convey("When parse valid flags", func() { - f, err := ParseCompileFlag("ifemopus") - - So(f, ShouldEqual, flags) - So(err, ShouldBeNil) - }) - - Convey("When parse invalid flags", func() { - f, err := ParseCompileFlag("abc") - - So(f, ShouldEqual, 0) - So(err, ShouldNotBeNil) - }) - }) -} - -type testAllocator struct { - memoryUsed int - memoryFreed []unsafe.Pointer -} - -func (a *testAllocator) alloc(size uint) unsafe.Pointer { - a.memoryUsed += int(size) - - return hsDefaultAlloc(size) -} - -func (a *testAllocator) free(ptr unsafe.Pointer) { - a.memoryFreed = append(a.memoryFreed, ptr) - - hsDefaultFree(ptr) -} - -func TestAllocator(t *testing.T) { - Convey("Given the host platform", t, func() { - platform, err := hsPopulatePlatform() - - So(platform, ShouldNotBeNil) - So(err, ShouldBeNil) - - a := &testAllocator{} - - Convey("Given a simple expression with allocator", func() { - So(hsSetMiscAllocator(a.alloc, a.free), ShouldBeNil) - - info, err := hsExpressionInfo("test", 0) - - So(info, ShouldNotBeNil) - So(info, ShouldResemble, &ExprInfo{ - MinWidth: 4, - MaxWidth: 4, - }) - So(err, ShouldBeNil) - - So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, 12) - - So(hsClearMiscAllocator(), ShouldBeNil) - }) - - Convey("Then create a stream database with allocator", func() { - So(hsSetDatabaseAllocator(a.alloc, a.free), ShouldBeNil) - - db, err := hsCompile("test", 0, StreamMode, platform) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Get the database size", func() { - size, err := hsDatabaseSize(db) - - So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, size) - So(err, ShouldBeNil) - }) - - Convey("Then create a scratch with allocator", func() { - So(hsSetScratchAllocator(a.alloc, a.free), ShouldBeNil) - - a.memoryUsed = 0 - - s, err := hsAllocScratch(db) - - So(s, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Get the scratch size", func() { - size, err := hsScratchSize(s) - - So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, size) - So(err, ShouldBeNil) - }) - - Convey("Then open a stream", func() { - So(hsSetStreamAllocator(a.alloc, a.free), ShouldBeNil) - - a.memoryUsed = 0 - - stream, err := hsOpenStream(db, 0) - - So(stream, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Get the stream size", func() { - size, err := hsStreamSize(db) - - So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, size) - So(err, ShouldBeNil) - }) - - h := &matchRecorder{} - - Convey("Then close stream with allocator", func() { - a.memoryFreed = nil - - So(hsCloseStream(stream, s, h.Handle, nil), ShouldBeNil) - - So(hsClearStreamAllocator(), ShouldBeNil) - }) - }) - - Convey("Then free scratch with allocator", func() { - a.memoryFreed = nil - - So(hsFreeScratch(s), ShouldBeNil) - - So(a.memoryFreed, ShouldResemble, []unsafe.Pointer{unsafe.Pointer(s)}) - - So(hsClearScratchAllocator(), ShouldBeNil) - }) - }) - - Convey("Then free database with allocator", func() { - a.memoryFreed = nil - - So(hsFreeDatabase(db), ShouldBeNil) - - So(a.memoryFreed, ShouldResemble, []unsafe.Pointer{unsafe.Pointer(db)}) - - So(hsClearDatabaseAllocator(), ShouldBeNil) - }) - }) - }) -} - -func TestDatabase(t *testing.T) { - Convey("Given a stream database", t, func() { - platform, err := hsPopulatePlatform() - - So(platform, ShouldNotBeNil) - So(err, ShouldBeNil) - - db, err := hsCompile("test", 0, StreamMode, platform) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Get the database info", func() { - info, err := hsDatabaseInfo(db) - - So(regexInfo.MatchString(info), ShouldBeTrue) - So(err, ShouldBeNil) - }) - - Convey("Get the database size", func() { - size, err := hsDatabaseSize(db) - - So(size, ShouldBeGreaterThan, 800) - So(err, ShouldBeNil) - }) - - Convey("Get the stream size", func() { - size, err := hsStreamSize(db) - - So(size, ShouldBeGreaterThan, 20) - So(err, ShouldBeNil) - }) - - Convey("Get the stream size from a block database", func() { - db, err := hsCompile("test", 0, BlockMode, platform) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - size, err := hsStreamSize(db) - - So(size, ShouldEqual, 0) - So(err, ShouldEqual, ErrDatabaseModeError) - }) - - Convey("When serialize database", func() { - data, err := hsSerializeDatabase(db) - - So(data, ShouldNotBeNil) - So(len(data), ShouldBeGreaterThan, 800) - So(err, ShouldBeNil) - - Convey("Get the database info", func() { - info, err := hsSerializedDatabaseInfo(data) - - So(regexInfo.MatchString(info), ShouldBeTrue) - So(err, ShouldBeNil) - }) - - Convey("Get the database size", func() { - size, err := hsSerializedDatabaseSize(data) - - So(size, ShouldBeGreaterThan, 800) - So(err, ShouldBeNil) - }) - - Convey("Then deserialize database", func() { - db, err := hsDeserializeDatabase(data) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Get the database info", func() { - info, err := hsDatabaseInfo(db) - - So(regexInfo.MatchString(info), ShouldBeTrue) - So(err, ShouldBeNil) - }) - }) - - Convey("Then deserialize database to memory", func() { - buf := make([]byte, 1000) - db := hsDatabase(unsafe.Pointer(&buf[0])) - - So(hsDeserializeDatabaseAt(data, db), ShouldBeNil) - - Convey("Get the database info", func() { - info, err := hsDatabaseInfo(db) - - So(regexInfo.MatchString(info), ShouldBeTrue) - So(err, ShouldBeNil) - }) - }) - }) - - So(hsFreeDatabase(db), ShouldBeNil) - }) -} - -func TestCompileAPI(t *testing.T) { - Convey("Given a host platform", t, func() { - platform, err := hsPopulatePlatform() - - So(platform, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("When Compile a unsupported expression", func() { - Convey("Then compile as stream", func() { - db, err := hsCompile(`\R`, 0, StreamMode, platform) - - So(db, ShouldBeNil) - So(err, ShouldNotBeNil) - So(err.Error(), ShouldEqual, `\R at index 0 not supported.`) - }) - - Convey("Then compile as vector", func() { - db, err := hsCompileMulti([]*Pattern{ - NewPattern(`\R`, Caseless), - }, BlockMode, platform) - - So(db, ShouldBeNil) - So(err, ShouldNotBeNil) - So(err.Error(), ShouldEqual, `\R at index 0 not supported.`) - }) - - Convey("Then compile as extended vector", func() { - db, err := hsCompileMulti([]*Pattern{ - NewPattern(`\R`, Caseless).WithExt(MinOffset(10)), - }, VectoredMode, platform) - - So(db, ShouldBeNil) - So(err, ShouldNotBeNil) - So(err.Error(), ShouldEqual, `\R at index 0 not supported.`) - }) - }) - - Convey("Compile an empty expression", func() { - db, err := hsCompile("", 0, StreamMode, platform) - - So(db, ShouldBeNil) - So(err, ShouldNotBeNil) - So(err.Error(), ShouldEqual, "Pattern matches empty buffer; use HS_FLAG_ALLOWEMPTY to enable support.") - - So(hsFreeDatabase(db), ShouldBeNil) - }) - - Convey("Compile multi expressions", func() { - db, err := hsCompileMulti([]*Pattern{ - NewPattern(`^\w+`, 0), - NewPattern(`\d+`, 0), - NewPattern(`\s+`, 0), - }, StreamMode, platform) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Get the database info", func() { - info, err := hsDatabaseInfo(db) - - So(regexInfo.MatchString(info), ShouldBeTrue) - So(err, ShouldBeNil) - }) - - So(hsFreeDatabase(db), ShouldBeNil) - }) - - Convey("Compile multi expressions with extension", func() { - db, err := hsCompileMulti([]*Pattern{ - NewPattern(`^\w+`, 0).WithExt(MinOffset(10)), - NewPattern(`\d+`, 0).WithExt(MaxOffset(10)), - NewPattern(`\s+`, 0).WithExt(MinLength(10)), - }, StreamMode, platform) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Get the database info", func() { - info, err := hsDatabaseInfo(db) - - So(regexInfo.MatchString(info), ShouldBeTrue) - So(err, ShouldBeNil) - }) - - So(hsFreeDatabase(db), ShouldBeNil) - }) - }) -} - -func TestExpression(t *testing.T) { - Convey("Given a simple expression", t, func() { - info, err := hsExpressionInfo("test", 0) - - So(info, ShouldNotBeNil) - So(info, ShouldResemble, &ExprInfo{ - MinWidth: 4, - MaxWidth: 4, - }) - So(err, ShouldBeNil) - }) - - Convey("Given a credit card expression", t, func() { - info, err := hsExpressionInfo(CreditCard, 0) - - So(info, ShouldNotBeNil) - So(info, ShouldResemble, &ExprInfo{ - MinWidth: 13, - MaxWidth: 16, - }) - So(err, ShouldBeNil) - }) - - Convey("Given a expression match eod", t, func() { - info, err := hsExpressionInfo("test$", 0) - - So(info, ShouldNotBeNil) - So(info, ShouldResemble, &ExprInfo{ - MinWidth: 4, - MaxWidth: 4, - ReturnUnordered: true, - AtEndOfData: true, - OnlyAtEndOfData: true, - }) - So(err, ShouldBeNil) - }) -} - -func TestScratch(t *testing.T) { - Convey("Given a block database", t, func() { - platform, err := hsPopulatePlatform() - - So(platform, ShouldNotBeNil) - So(err, ShouldBeNil) - - db, err := hsCompile("test", 0, BlockMode, platform) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Allocate a scratch", func() { - s, err := hsAllocScratch(db) - - So(s, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Get the scratch size", func() { - size, err := hsScratchSize(s) - - So(size, ShouldBeGreaterThan, 1024) - So(size, ShouldBeLessThan, 4096) - So(err, ShouldBeNil) - - Convey("Clone the scratch", func() { - s2, err := hsCloneScratch(s) - - So(s2, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Cloned scrash should have same size", func() { - size2, err := hsScratchSize(s2) - - So(size2, ShouldEqual, size) - So(err, ShouldBeNil) - }) - - So(hsFreeScratch(s2), ShouldBeNil) - }) - - Convey("Reallocate the scratch with another database", func() { - db2, err := hsCompile(EmailAddress, 0, BlockMode, platform) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - So(hsReallocScratch(db2, &s), ShouldBeNil) - - size2, err := hsScratchSize(s) - - So(size2, ShouldBeGreaterThan, size) - So(err, ShouldBeNil) - - So(hsFreeDatabase(db2), ShouldBeNil) - }) - }) - - So(hsFreeScratch(s), ShouldBeNil) - }) - - So(hsFreeDatabase(db), ShouldBeNil) - }) -} - -func TestBlockScan(t *testing.T) { - Convey("Given a block database", t, func() { - platform, err := hsPopulatePlatform() - - So(platform, ShouldNotBeNil) - So(err, ShouldBeNil) - - db, err := hsCompile("test", 0, BlockMode, platform) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - s, err := hsAllocScratch(db) - - So(s, ShouldNotBeNil) - So(err, ShouldBeNil) - - h := &matchRecorder{} - - Convey("Scan block with pattern", func() { - So(hsScan(db, []byte("abctestdef"), 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldResemble, []matchEvent{{0, 0, 7, 0}}) - }) - - Convey("Scan block without pattern", func() { - So(hsScan(db, []byte("abcdef"), 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldBeEmpty) - }) - - Convey("Scan block with multi pattern", func() { - So(hsScan(db, []byte("abctestdeftest"), 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldResemble, []matchEvent{{0, 0, 14, 0}}) - }) - - Convey("Scan block with multi pattern but terminated", func() { - h.err = ErrScanTerminated - - So(hsScan(db, []byte("abctestdeftest"), 0, s, h.Handle, nil), ShouldEqual, ErrScanTerminated) - So(h.matched, ShouldResemble, []matchEvent{{0, 0, 7, 0}}) - }) - - Convey("Scan empty buffers", func() { - So(hsScan(db, nil, 0, s, h.Handle, nil), ShouldEqual, ErrInvalid) - So(hsScan(db, []byte(""), 0, s, h.Handle, nil), ShouldBeNil) - }) - - So(hsFreeScratch(s), ShouldBeNil) - So(hsFreeDatabase(db), ShouldBeNil) - }) -} - -func TestVectorScan(t *testing.T) { - Convey("Given a block database", t, func() { - platform, err := hsPopulatePlatform() - - So(platform, ShouldNotBeNil) - So(err, ShouldBeNil) - - db, err := hsCompile("test", 0, VectoredMode, platform) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - s, err := hsAllocScratch(db) - - So(s, ShouldNotBeNil) - So(err, ShouldBeNil) - - h := &matchRecorder{} - - Convey("Scan multi block with pattern", func() { - So(hsScanVector(db, [][]byte{[]byte("abctestdef"), []byte("abcdef")}, 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldResemble, []matchEvent{{0, 0, 7, 0}}) - }) - - Convey("Scan multi block without pattern", func() { - So(hsScanVector(db, [][]byte{[]byte("123456"), []byte("abcdef")}, 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldBeEmpty) - }) - - Convey("Scan multi block with multi pattern", func() { - So(hsScanVector(db, [][]byte{[]byte("abctestdef"), []byte("123test456")}, 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldResemble, []matchEvent{{0, 0, 17, 0}}) - }) - - Convey("Scan multi block with multi pattern but terminated", func() { - h.err = ErrScanTerminated - - So(hsScanVector(db, [][]byte{[]byte("abctestdef"), []byte("123test456")}, 0, s, h.Handle, nil), - ShouldEqual, ErrScanTerminated) - So(h.matched, ShouldResemble, []matchEvent{{0, 0, 7, 0}}) - }) - - Convey("Scan empty buffers", func() { - So(hsScanVector(db, nil, 0, s, h.Handle, nil), ShouldEqual, ErrInvalid) - So(hsScanVector(db, [][]byte{}, 0, s, h.Handle, nil), ShouldBeNil) - So(hsScanVector(db, [][]byte{[]byte(""), []byte("")}, 0, s, h.Handle, nil), ShouldBeNil) - }) - - So(hsFreeScratch(s), ShouldBeNil) - }) -} - -func TestStreamScan(t *testing.T) { - Convey("Given a stream database", t, func() { - platform, err := hsPopulatePlatform() - - So(platform, ShouldNotBeNil) - So(err, ShouldBeNil) - - db, err := hsCompile("test", 0, StreamMode, platform) - - So(db, ShouldNotBeNil) - So(err, ShouldBeNil) - - s, err := hsAllocScratch(db) - - So(s, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("Then open a stream", func() { - stream, err := hsOpenStream(db, 0) - - So(stream, ShouldNotBeNil) - So(err, ShouldBeNil) - - h := &matchRecorder{} - - Convey("Then scan a simple stream with first part", func() { - So(hsScanStream(stream, []byte("abcte"), 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldBeNil) - - Convey("When scan second part, should be matched", func() { - So(hsScanStream(stream, []byte("stdef"), 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldResemble, []matchEvent{{0, 0, 7, 0}}) - }) - - Convey("Then copy the stream", func() { - stream2, err := hsCopyStream(stream) - - So(stream2, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("When copied stream2 scan the second part, should be matched", func() { - So(hsScanStream(stream2, []byte("stdef"), 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldResemble, []matchEvent{{0, 0, 7, 0}}) - - Convey("When copied stream2 scan the second part again, should not be matched", func() { - h.matched = nil - So(hsScanStream(stream2, []byte("stdef"), 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldBeNil) - - Convey("When copy and reset stream2", func() { - So(hsResetAndCopyStream(stream2, stream, s, h.Handle, nil), ShouldBeNil) - - Convey("When copied and reset stream2 scan the second part again, should be matched", func() { - h.matched = nil - So(hsScanStream(stream2, []byte("stdef"), 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldResemble, []matchEvent{{0, 0, 7, 0}}) - }) - }) - }) - }) - - So(hsCloseStream(stream2, s, h.Handle, nil), ShouldBeNil) - }) - - Convey("Then reset the stream", func() { - So(hsResetStream(stream, 0, s, h.Handle, nil), ShouldBeNil) - - Convey("When scan the second part, should not be matched", func() { - So(hsScanStream(stream, []byte("stdef"), 0, s, h.Handle, nil), ShouldBeNil) - So(h.matched, ShouldBeNil) - }) - - Convey("When scan empty buffers", func() { - So(hsScanStream(stream, nil, 0, s, h.Handle, nil), ShouldEqual, ErrInvalid) - So(hsScanStream(stream, []byte(""), 0, s, h.Handle, nil), ShouldBeNil) - }) - }) - }) - - So(hsCloseStream(stream, s, h.Handle, nil), ShouldBeNil) - }) - - So(hsFreeScratch(s), ShouldBeNil) - }) -} diff --git a/hyperscan/internal_v54.go b/hyperscan/internal_v54.go deleted file mode 100644 index f7b6ecb..0000000 --- a/hyperscan/internal_v54.go +++ /dev/null @@ -1,18 +0,0 @@ -//go:build hyperscan_v54 -// +build hyperscan_v54 - -package hyperscan - -/* -#include -*/ -import "C" - -const ( - AVX512VBMI CpuFeature = C.HS_CPU_FEATURES_AVX512VBMI // AVX512VBMI is a CPU features flag indicates that the target platform supports Intel(R) Advanced Vector Extensions 512 Vector Byte Manipulation Instructions (Intel(R) AVX512VBMI) -) - -const ( - Icelake TuneFlag = C.HS_TUNE_FAMILY_ICL // Icelake indicates that the compiled database should be tuned for the Icelake microarchitecture. - IcelakeServer TuneFlag = C.HS_TUNE_FAMILY_ICX // IcelakeServer indicates that the compiled database should be tuned for the Icelake Server microarchitecture. -) diff --git a/hyperscan/literal.go b/hyperscan/literal.go new file mode 100644 index 0000000..cfc6e44 --- /dev/null +++ b/hyperscan/literal.go @@ -0,0 +1,214 @@ +//go:build !hyperscan_v4 +// +build !hyperscan_v4 + +package hyperscan + +import ( + "fmt" + "strconv" + "strings" + + "github.com/flier/gohs/internal/hs" +) + +type Literals []*Literal + +// Pure literal is a special case of regular expression. +// A character sequence is regarded as a pure literal if and +// only if each character is read and interpreted independently. +// No syntax association happens between any adjacent characters. +// nolint: golint,revive,stylecheck +type Literal struct { + // The expression to parse. + Expression + // Flags which modify the behaviour of the expression. + Flags CompileFlag + // The ID number to be associated with the corresponding pattern + Id int + *ExprInfo +} + +// NewLiteral returns a new Literal base on expression and compile flags. +func NewLiteral(expr string, flags ...CompileFlag) *Literal { + var v CompileFlag + for _, f := range flags { + v |= f + } + return &Literal{Expression: Expression(expr), Flags: v} +} + +// IsValid validate the literal contains a pure literal. +func (lit *Literal) IsValid() bool { + _, err := lit.Info() + + return err == nil +} + +// Provides information about a regular expression. +func (lit *Literal) Info() (*ExprInfo, error) { + if lit.ExprInfo == nil { + info, err := hs.ExpressionInfo(string(lit.Expression), lit.Flags) + if err != nil { + return nil, err // nolint: wrapcheck + } + + lit.ExprInfo = info + } + + return lit.ExprInfo, nil +} + +func (lit *Literal) String() string { + var b strings.Builder + + if lit.Id > 0 { + fmt.Fprintf(&b, "%d:", lit.Id) + } + + fmt.Fprintf(&b, "/%s/%s", lit.Expression, lit.Flags) + + return b.String() +} + +/* +Parse literal from a formated string + + :// + +For example, the following literal will match `test` in the caseless and multi-lines mode + + /test/im + +*/ +func ParseLiteral(s string) (*Literal, error) { + var lit Literal + + i := strings.Index(s, ":/") + j := strings.LastIndex(s, "/") + if i > 0 && j > i+1 { + id, err := strconv.Atoi(s[:i]) + if err != nil { + return nil, fmt.Errorf("invalid pattern id `%s`, %w", s[:i], ErrInvalid) + } + lit.Id = id + s = s[i+1:] + } + + if n := strings.LastIndex(s, "/"); n > 1 && strings.HasPrefix(s, "/") { + lit.Expression = Expression(s[1:n]) + s = s[n+1:] + + flags, err := ParseCompileFlag(s) + if err != nil { + return nil, fmt.Errorf("invalid pattern flags `%s`, %w", s, err) + } + lit.Flags = flags + } else { + lit.Expression = Expression(s) + } + + info, err := hs.ExpressionInfo(string(lit.Expression), lit.Flags) + if err != nil { + return nil, fmt.Errorf("invalid pattern `%s`, %w", lit.Expression, err) + } + lit.ExprInfo = info + + return &lit, nil +} + +func (lit *Literal) Literal() *hs.Literal { + return &hs.Literal{ + Expr: string(lit.Expression), + Flags: lit.Flags, + ID: lit.Id, + ExprInfo: lit.ExprInfo, + } +} + +func (lit *Literal) Literals() (r []*hs.Literal) { + return []*hs.Literal{lit.Literal()} +} + +func (lit *Literal) Build(mode ModeFlag) (Database, error) { + return lit.ForPlatform(mode, nil) +} + +func (lit *Literal) ForPlatform(mode ModeFlag, platform Platform) (Database, error) { + if mode == 0 { + mode = BlockMode + } else if mode == StreamMode { + som := (lit.Flags & SomLeftMost) == SomLeftMost + + if som && mode&(SomHorizonSmallMode|SomHorizonMediumMode|SomHorizonLargeMode) == 0 { + mode |= SomHorizonSmallMode + } + } + + p, _ := platform.(*hs.PlatformInfo) + + db, err := hs.CompileLit(string(lit.Expression), lit.Flags, mode, p) + if err != nil { + return nil, err // nolint: wrapcheck + } + + switch mode & hs.ModeMask { + case StreamMode: + return newStreamDatabase(db), nil + case VectoredMode: + return newVectoredDatabase(db), nil + case BlockMode: + return newBlockDatabase(db), nil + } + + return nil, fmt.Errorf("mode %d, %w", mode, ErrInvalid) +} + +func (literals Literals) Literals() (r []*hs.Literal) { + r = make([]*hs.Literal, len(literals)) + + for i, lit := range literals { + r[i] = lit.Literal() + } + + return +} + +func (literals Literals) Build(mode ModeFlag) (Database, error) { + return literals.ForPlatform(mode, nil) +} + +func (literals Literals) ForPlatform(mode ModeFlag, platform Platform) (Database, error) { + if mode == 0 { + mode = BlockMode + } else if mode == StreamMode { + som := false + + for _, lit := range literals { + if (lit.Flags & SomLeftMost) == SomLeftMost { + som = true + } + } + + if som && mode&(SomHorizonSmallMode|SomHorizonMediumMode|SomHorizonLargeMode) == 0 { + mode |= SomHorizonSmallMode + } + } + + p, _ := platform.(*hs.PlatformInfo) + + db, err := hs.CompileLitMulti(literals, mode, p) + if err != nil { + return nil, err // nolint: wrapcheck + } + + switch mode & hs.ModeMask { + case StreamMode: + return newStreamDatabase(db), nil + case VectoredMode: + return newVectoredDatabase(db), nil + case BlockMode: + return newBlockDatabase(db), nil + } + + return nil, fmt.Errorf("mode %d, %w", mode, ErrInvalid) +} diff --git a/hyperscan/literal_test.go b/hyperscan/literal_test.go new file mode 100644 index 0000000..e675a43 --- /dev/null +++ b/hyperscan/literal_test.go @@ -0,0 +1,98 @@ +//go:build !hyperscan_v4 +// +build !hyperscan_v4 + +package hyperscan_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/hyperscan" +) + +//nolint:funlen +func TestLiteral(t *testing.T) { + Convey("Give a literal", t, func() { + Convey("When parse with flags", func() { + p, err := hyperscan.ParseLiteral(`/test/im`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "test") + So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) + + So(string(p.Expression), ShouldEqual, "test") + So(p.String(), ShouldEqual, `/test/im`) + + Convey("When literal contains regular grammar", func() { + p, err := hyperscan.ParseLiteral(`/te?st/im`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "te?st") + So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) + + So(p.String(), ShouldEqual, "/te?st/im") + }) + }) + + Convey("When parse literal with id and flags", func() { + p, err := hyperscan.ParseLiteral("3:/foobar/iu") + + So(err, ShouldBeNil) + So(p.Id, ShouldEqual, 3) + So(p.Expression, ShouldEqual, "foobar") + So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.Utf8Mode) + }) + + Convey("When parse with a lot of flags", func() { + p, err := hyperscan.ParseLiteral(`/test/ismoeupf`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "test") + So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.DotAll|hyperscan.MultiLine|hyperscan.SingleMatch| + hyperscan.AllowEmpty|hyperscan.Utf8Mode|hyperscan.UnicodeProperty|hyperscan.PrefilterMode) + + So(p.Flags.String(), ShouldEqual, "8HPVWims") + So(p.String(), ShouldEqual, "/test/8HPVWims") + }) + + Convey("When parse without flags", func() { + p, err := hyperscan.ParseLiteral(`test`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "test") + So(p.Flags, ShouldEqual, 0) + + Convey("When literal contains regular grammar", func() { + p, err := hyperscan.ParseLiteral(`/te?st/im`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "te?st") + So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) + + So(p.String(), ShouldEqual, "/te?st/im") + }) + }) + + Convey("When literal is valid", func() { + p := hyperscan.NewLiteral("test") + + info, err := p.Info() + + So(err, ShouldBeNil) + So(info, ShouldNotBeNil) + So(info, ShouldResemble, &hyperscan.ExprInfo{MinWidth: 4, MaxWidth: 4}) + So(p.IsValid(), ShouldBeTrue) + }) + + Convey("When quote a string", func() { + So(hyperscan.Quote("test"), ShouldEqual, "`test`") + So(hyperscan.Quote("`can't backquote this`"), ShouldEqual, "\"`can't backquote this`\"") + }) + }) +} diff --git a/hyperscan/pattern.go b/hyperscan/pattern.go new file mode 100644 index 0000000..4112e68 --- /dev/null +++ b/hyperscan/pattern.go @@ -0,0 +1,381 @@ +package hyperscan + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" + + "github.com/flier/gohs/internal/hs" +) + +// ExprInfo containing information related to an expression. +type ExprInfo = hs.ExprInfo + +// ExtFlag are used in ExprExt.Flags to indicate which fields are used. +type ExtFlag = hs.ExtFlag + +const ( + // ExtMinOffset is a flag indicating that the ExprExt.MinOffset field is used. + ExtMinOffset ExtFlag = hs.ExtMinOffset + // ExtMaxOffset is a flag indicating that the ExprExt.MaxOffset field is used. + ExtMaxOffset ExtFlag = hs.ExtMaxOffset + // ExtMinLength is a flag indicating that the ExprExt.MinLength field is used. + ExtMinLength ExtFlag = hs.ExtMinLength + // ExtEditDistance is a flag indicating that the ExprExt.EditDistance field is used. + ExtEditDistance ExtFlag = hs.ExtEditDistance + // ExtHammingDistance is a flag indicating that the ExprExt.HammingDistance field is used. + ExtHammingDistance ExtFlag = hs.ExtHammingDistance +) + +// Ext is a option containing additional parameters related to an expression. +type Ext func(ext *ExprExt) + +// MinOffset given the minimum end offset in the data stream at which this expression should match successfully. +func MinOffset(n uint64) Ext { + return func(ext *ExprExt) { + ext.Flags |= ExtMinOffset + ext.MinOffset = n + } +} + +// MaxOffset given the maximum end offset in the data stream at which this expression should match successfully. +func MaxOffset(n uint64) Ext { + return func(ext *ExprExt) { + ext.Flags |= ExtMaxOffset + ext.MaxOffset = n + } +} + +// MinLength given the minimum match length (from start to end) required to successfully match this expression. +func MinLength(n uint64) Ext { + return func(ext *ExprExt) { + ext.Flags |= ExtMinLength + ext.MinLength = n + } +} + +// EditDistance allow patterns to approximately match within this edit distance. +func EditDistance(n uint32) Ext { + return func(ext *ExprExt) { + ext.Flags |= ExtEditDistance + ext.EditDistance = n + } +} + +// HammingDistance allow patterns to approximately match within this Hamming distance. +func HammingDistance(n uint32) Ext { + return func(ext *ExprExt) { + ext.Flags |= ExtHammingDistance + ext.HammingDistance = n + } +} + +// ExprExt is a structure containing additional parameters related to an expression. +type ExprExt hs.ExprExt + +func NewExprExt(exts ...Ext) (ext *ExprExt) { + if len(exts) == 0 { + return + } + + ext = new(ExprExt) + + for _, f := range exts { + f(ext) + } + + return ext +} + +// With specifies the additional parameters related to an expression. +func (ext *ExprExt) With(exts ...Ext) *ExprExt { + for _, f := range exts { + f(ext) + } + + return ext +} + +func (ext *ExprExt) String() string { + var values []string + + if (ext.Flags & ExtMinOffset) == ExtMinOffset { + values = append(values, fmt.Sprintf("min_offset=%d", ext.MinOffset)) + } + + if (ext.Flags & ExtMaxOffset) == ExtMaxOffset { + values = append(values, fmt.Sprintf("max_offset=%d", ext.MaxOffset)) + } + + if (ext.Flags & ExtMinLength) == ExtMinLength { + values = append(values, fmt.Sprintf("min_length=%d", ext.MinLength)) + } + + if (ext.Flags & ExtEditDistance) == ExtEditDistance { + values = append(values, fmt.Sprintf("edit_distance=%d", ext.EditDistance)) + } + + if (ext.Flags & ExtHammingDistance) == ExtHammingDistance { + values = append(values, fmt.Sprintf("hamming_distance=%d", ext.HammingDistance)) + } + + return "{" + strings.Join(values, ",") + "}" +} + +const keyValuePair = 2 + +// ParseExprExt parse containing additional parameters from string. +func ParseExprExt(s string) (ext *ExprExt, err error) { + ext = new(ExprExt) + + if strings.HasPrefix(s, "{") && strings.HasSuffix(s, "}") { + s = strings.TrimSuffix(strings.TrimPrefix(s, "{"), "}") + } + + for _, s := range strings.Split(s, ",") { + parts := strings.SplitN(s, "=", keyValuePair) + + if len(parts) != keyValuePair { + continue + } + + key := strings.ToLower(parts[0]) + value := parts[1] + + var n int + + if n, err = strconv.Atoi(value); err != nil { + return + } + + switch key { + case "min_offset": + ext.Flags |= ExtMinOffset + ext.MinOffset = uint64(n) + + case "max_offset": + ext.Flags |= ExtMaxOffset + ext.MaxOffset = uint64(n) + + case "min_length": + ext.Flags |= ExtMinLength + ext.MinLength = uint64(n) + + case "edit_distance": + ext.Flags |= ExtEditDistance + ext.EditDistance = uint32(n) + + case "hamming_distance": + ext.Flags |= ExtHammingDistance + ext.HammingDistance = uint32(n) + } + } + + return // nolint: nakedret +} + +// Expression of pattern. +type Expression string + +func (e Expression) String() string { return string(e) } + +// Pattern is a matching pattern. +// nolint: golint,revive,stylecheck +type Pattern struct { + // The expression to parse. + Expression + // Flags which modify the behaviour of the expression. + Flags CompileFlag + // The ID number to be associated with the corresponding pattern + Id int + info *ExprInfo + ext *ExprExt +} + +// NewPattern returns a new pattern base on expression and compile flags. +func NewPattern(expr Expression, flags CompileFlag, exts ...Ext) *Pattern { + return &Pattern{ + Expression: expr, + Flags: flags, + ext: NewExprExt(exts...), + } +} + +func (p *Pattern) Pattern() *hs.Pattern { + return &hs.Pattern{ + Expr: string(p.Expression), + Flags: p.Flags, + ID: p.Id, + Ext: (*hs.ExprExt)(p.ext), + } +} + +func (p *Pattern) Patterns() []*hs.Pattern { + return []*hs.Pattern{p.Pattern()} +} + +// IsValid validate the pattern contains a regular expression. +func (p *Pattern) IsValid() bool { + _, err := p.Info() + + return err == nil +} + +// Info provides information about a regular expression. +func (p *Pattern) Info() (*ExprInfo, error) { + if p.info == nil { + info, err := hs.ExpressionInfo(string(p.Expression), p.Flags) + if err != nil { + return nil, err // nolint: wrapcheck + } + + p.info = info + } + + return p.info, nil +} + +// WithExt is used to set the additional parameters related to an expression. +func (p *Pattern) WithExt(exts ...Ext) *Pattern { + if p.ext == nil { + p.ext = new(ExprExt) + } + + p.ext.With(exts...) + + return p +} + +// Ext provides additional parameters related to an expression. +func (p *Pattern) Ext() (*ExprExt, error) { + if p.ext == nil { + ext, info, err := hs.ExpressionExt(string(p.Expression), p.Flags) + if err != nil { + return nil, err // nolint: wrapcheck + } + + p.ext = (*ExprExt)(ext) + p.info = info + } + + return p.ext, nil +} + +func (p *Pattern) String() string { + var b strings.Builder + + if p.Id > 0 { + fmt.Fprintf(&b, "%d:", p.Id) + } + + fmt.Fprintf(&b, "/%s/%s", p.Expression, p.Flags) + + if p.ext != nil { + b.WriteString(p.ext.String()) + } + + return b.String() +} + +/* +ParsePattern parse pattern from a formated string. + + :// + +For example, the following pattern will match `test` in the caseless and multi-lines mode + + /test/im + +*/ +func ParsePattern(s string) (*Pattern, error) { + var p Pattern + + i := strings.Index(s, ":/") + j := strings.LastIndex(s, "/") + + if i > 0 && j > i+1 { + id, err := strconv.Atoi(s[:i]) + if err != nil { + return nil, fmt.Errorf("pattern id `%s`, %w", s[:i], ErrInvalid) + } + + p.Id = id + s = s[i+1:] + } + + if n := strings.LastIndex(s, "/"); n > 1 && strings.HasPrefix(s, "/") { + p.Expression = Expression(s[1:n]) + s = s[n+1:] + + if n = strings.Index(s, "{"); n > 0 && strings.HasSuffix(s, "}") { + ext, err := ParseExprExt(s[n:]) + if err != nil { + return nil, fmt.Errorf("expression extensions `%s`, %w", s[n:], err) + } + + p.ext = ext + s = s[:n] + } + + flags, err := ParseCompileFlag(s) + if err != nil { + return nil, fmt.Errorf("pattern flags `%s`, %w", s, err) + } + + p.Flags = flags + } else { + p.Expression = Expression(s) + } + + info, err := hs.ExpressionInfo(string(p.Expression), p.Flags) + if err != nil { + return nil, fmt.Errorf("pattern `%s`, %w", p.Expression, err) + } + + p.info = info + + return &p, nil +} + +// Patterns is a set of matching patterns. +type Patterns []*Pattern + +// ParsePatterns parse lines as `Patterns`. +func ParsePatterns(r io.Reader) (patterns Patterns, err error) { + s := bufio.NewScanner(r) + + for s.Scan() { + line := strings.TrimSpace(s.Text()) + + if line == "" { + // skip empty line + continue + } + + if strings.HasPrefix(line, "#") { + // skip comment + continue + } + + p, err := ParsePattern(line) + if err != nil { + return nil, err + } + + patterns = append(patterns, p) + } + + return +} + +func (p Patterns) Patterns() (r []*hs.Pattern) { + r = make([]*hs.Pattern, len(p)) + + for i, pat := range p { + r[i] = pat.Pattern() + } + + return +} diff --git a/hyperscan/pattern_test.go b/hyperscan/pattern_test.go new file mode 100644 index 0000000..15f2532 --- /dev/null +++ b/hyperscan/pattern_test.go @@ -0,0 +1,117 @@ +package hyperscan_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/hyperscan" +) + +//nolint:funlen +func TestPattern(t *testing.T) { + Convey("Give a pattern", t, func() { + Convey("When parse with flags", func() { + p, err := hyperscan.ParsePattern(`/test/im`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "test") + So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) + + So(string(p.Expression), ShouldEqual, "test") + So(p.String(), ShouldEqual, `/test/im`) + + Convey("When pattern contains forward slash", func() { + p, err := hyperscan.ParsePattern(`/te/st/im`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "te/st") + So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) + + So(p.String(), ShouldEqual, "/te/st/im") + }) + }) + + Convey("When parse pattern with id and flags", func() { + p, err := hyperscan.ParsePattern("3:/foobar/iu") + + So(err, ShouldBeNil) + So(p.Id, ShouldEqual, 3) + So(p.Expression, ShouldEqual, "foobar") + So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.Utf8Mode) + }) + + Convey("When parse pattern with id, flags and extensions", func() { + p, err := hyperscan.ParsePattern("3:/foobar/iu{min_offset=4,min_length=8}") + So(err, ShouldBeNil) + So(p.Id, ShouldEqual, 3) + So(p.Expression, ShouldEqual, "foobar") + So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.Utf8Mode) + + ext, err := p.Ext() + So(err, ShouldBeNil) + So(ext, ShouldResemble, new(hyperscan.ExprExt).With(hyperscan.MinOffset(4), hyperscan.MinLength(8))) + + So(p.String(), ShouldEqual, "3:/foobar/8i{min_offset=4,min_length=8}") + }) + + Convey("When parse with a lot of flags", func() { + p, err := hyperscan.ParsePattern(`/test/ismoeupf`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "test") + So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.DotAll|hyperscan.MultiLine|hyperscan.SingleMatch| + hyperscan.AllowEmpty|hyperscan.Utf8Mode|hyperscan.UnicodeProperty|hyperscan.PrefilterMode) + + So(p.Flags.String(), ShouldEqual, "8HPVWims") + So(p.String(), ShouldEqual, "/test/8HPVWims") + }) + + Convey("When parse without flags", func() { + p, err := hyperscan.ParsePattern(`test`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "test") + So(p.Flags, ShouldEqual, 0) + + Convey("When pattern contains forward slash", func() { + p, err := hyperscan.ParsePattern(`te/st`) + + So(err, ShouldBeNil) + So(p, ShouldNotBeNil) + So(p.Expression, ShouldEqual, "te/st") + So(p.Flags, ShouldEqual, 0) + }) + }) + + Convey("When pattern is valid", func() { + p := hyperscan.Pattern{Expression: "test"} + + info, err := p.Info() + + So(err, ShouldBeNil) + So(info, ShouldNotBeNil) + So(info, ShouldResemble, &hyperscan.ExprInfo{MinWidth: 4, MaxWidth: 4}) + So(p.IsValid(), ShouldBeTrue) + }) + + Convey("When pattern is invalid", func() { + p := hyperscan.Pattern{Expression: `\R`} + + info, err := p.Info() + + So(err, ShouldNotBeNil) + So(info, ShouldBeNil) + So(p.IsValid(), ShouldBeFalse) + }) + + Convey("When quote a string", func() { + So(hyperscan.Quote("test"), ShouldEqual, "`test`") + So(hyperscan.Quote("`can't backquote this`"), ShouldEqual, "\"`can't backquote this`\"") + }) + }) +} diff --git a/hyperscan/platform.go b/hyperscan/platform.go new file mode 100644 index 0000000..a870872 --- /dev/null +++ b/hyperscan/platform.go @@ -0,0 +1,56 @@ +package hyperscan + +import "github.com/flier/gohs/internal/hs" + +type TuneFlag = hs.TuneFlag + +const ( + // Generic indicates that the compiled database should not be tuned for any particular target platform. + Generic TuneFlag = hs.Generic + // SandyBridge indicates that the compiled database should be tuned for the Sandy Bridge microarchitecture. + SandyBridge TuneFlag = hs.SandyBridge + // IvyBridge indicates that the compiled database should be tuned for the Ivy Bridge microarchitecture. + IvyBridge TuneFlag = hs.IvyBridge + // Haswell indicates that the compiled database should be tuned for the Haswell microarchitecture. + Haswell TuneFlag = hs.Haswell + // Silvermont indicates that the compiled database should be tuned for the Silvermont microarchitecture. + Silvermont TuneFlag = hs.Silvermont + // Broadwell indicates that the compiled database should be tuned for the Broadwell microarchitecture. + Broadwell TuneFlag = hs.Broadwell + // Skylake indicates that the compiled database should be tuned for the Skylake microarchitecture. + Skylake TuneFlag = hs.Skylake + // SkylakeServer indicates that the compiled database should be tuned for the Skylake Server microarchitecture. + SkylakeServer TuneFlag = hs.SkylakeServer + // Goldmont indicates that the compiled database should be tuned for the Goldmont microarchitecture. + Goldmont TuneFlag = hs.Goldmont +) + +// CpuFeature is the CPU feature support flags. +type CpuFeature = hs.CpuFeature // nolint: golint,stylecheck,revive + +const ( + // AVX2 is a CPU features flag indicates that the target platform supports AVX2 instructions. + AVX2 CpuFeature = hs.AVX2 + // AVX512 is a CPU features flag indicates that the target platform supports AVX512 instructions, + // specifically AVX-512BW. Using AVX512 implies the use of AVX2. + AVX512 CpuFeature = hs.AVX512 +) + +// Platform is a type containing information on the target platform. +type Platform interface { + // Information about the target platform which may be used to guide the optimisation process of the compile. + Tune() TuneFlag + + // Relevant CPU features available on the target platform + CpuFeatures() CpuFeature +} + +// NewPlatform create a new platform information on the target platform. +func NewPlatform(tune TuneFlag, cpu CpuFeature) Platform { return hs.NewPlatformInfo(tune, cpu) } + +// PopulatePlatform populates the platform information based on the current host. +func PopulatePlatform() Platform { + platform, _ := hs.PopulatePlatform() + + return platform +} diff --git a/hyperscan/platform_test.go b/hyperscan/platform_test.go new file mode 100644 index 0000000..a19a06d --- /dev/null +++ b/hyperscan/platform_test.go @@ -0,0 +1,21 @@ +package hyperscan_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/hyperscan" +) + +func TestPlatform(t *testing.T) { + Convey("Given a native platform", t, func() { + p := hyperscan.PopulatePlatform() + + So(p, ShouldNotBeNil) + So(p.Tune(), ShouldBeGreaterThan, hyperscan.Generic) + So(p.CpuFeatures(), ShouldBeGreaterThanOrEqualTo, 0) + + So(p, ShouldResemble, hyperscan.NewPlatform(p.Tune(), p.CpuFeatures())) + }) +} diff --git a/hyperscan/platform_v54.go b/hyperscan/platform_v54.go new file mode 100644 index 0000000..6f885b3 --- /dev/null +++ b/hyperscan/platform_v54.go @@ -0,0 +1,22 @@ +//go:build hyperscan_v54 +// +build hyperscan_v54 + +package hyperscan + +/* +#include +*/ +import "C" + +const ( + // AVX512VBMI is a CPU features flag indicates that the target platform + // supports Intel(R) Advanced Vector Extensions 512 Vector Byte Manipulation Instructions (Intel(R) AVX512VBMI) + AVX512VBMI CpuFeature = hs.AVX512VBMI +) + +const ( + // Icelake indicates that the compiled database should be tuned for the Icelake microarchitecture. + Icelake TuneFlag = hs.Icelake + // IcelakeServer indicates that the compiled database should be tuned for the Icelake Server microarchitecture. + IcelakeServer TuneFlag = hs.IcelakeServer +) diff --git a/hyperscan/runtime.go b/hyperscan/runtime.go index 7a3e066..8cc3aa7 100644 --- a/hyperscan/runtime.go +++ b/hyperscan/runtime.go @@ -2,69 +2,13 @@ package hyperscan import ( "errors" - "fmt" - "io" - "runtime" -) -const bufSize = 4096 + "github.com/flier/gohs/internal/hs" +) // ErrTooManyMatches means too many matches. var ErrTooManyMatches = errors.New("too many matches") -// Scratch is a Hyperscan scratch space. -type Scratch struct { - s hsScratch -} - -// NewScratch allocate a "scratch" space for use by Hyperscan. -// This is required for runtime use, and one scratch space per thread, -// or concurrent caller, is required. -func NewScratch(db Database) (*Scratch, error) { - s, err := hsAllocScratch(db.(database).Db()) - if err != nil { - return nil, err - } - - return &Scratch{s}, nil -} - -// NewManagedScratch is a wrapper for NewScratch that sets -// a finalizer on the Scratch instance so that memory is freed -// once the object is no longer in use. -func NewManagedScratch(db Database) (*Scratch, error) { - s, err := NewScratch(db) - if err != nil { - return nil, err - } - - runtime.SetFinalizer(s, func(scratch *Scratch) { - _ = scratch.Free() - }) - return s, nil -} - -// Size provides the size of the given scratch space. -func (s *Scratch) Size() (int, error) { return hsScratchSize(s.s) } - -// Realloc reallocate the scratch for another database. -func (s *Scratch) Realloc(db Database) error { - return hsReallocScratch(db.(database).Db(), &s.s) -} - -// Clone allocate a scratch space that is a clone of an existing scratch space. -func (s *Scratch) Clone() (*Scratch, error) { - cloned, err := hsCloneScratch(s.s) - if err != nil { - return nil, err - } - - return &Scratch{cloned}, nil -} - -// Free a scratch block previously allocated. -func (s *Scratch) Free() error { return hsFreeScratch(s.s) } - // MatchContext represents a match context. type MatchContext interface { Database() Database @@ -85,587 +29,7 @@ type MatchEvent interface { Flags() ScanFlag } -// MatchHandler handles match events. -type MatchHandler hsMatchEventHandler - -// BlockScanner is the block (non-streaming) regular expression scanner. -type BlockScanner interface { - // This is the function call in which the actual pattern matching takes place for block-mode pattern databases. - Scan(data []byte, scratch *Scratch, handler MatchHandler, context interface{}) error -} - -// BlockMatcher implements regular expression search. -type BlockMatcher interface { - // Find returns a slice holding the text of the leftmost match in b of the regular expression. - // A return value of nil indicates no match. - Find(data []byte) []byte - - // FindIndex returns a two-element slice of integers defining - // the location of the leftmost match in b of the regular expression. - // The match itself is at b[loc[0]:loc[1]]. A return value of nil indicates no match. - FindIndex(data []byte) []int - - // FindAll is the 'All' version of Find; it returns a slice of all successive matches of the expression, - // as defined by the 'All' description in the package comment. A return value of nil indicates no match. - FindAll(data []byte, n int) [][]byte - - // FindAllIndex is the 'All' version of FindIndex; it returns a slice of all successive matches of the expression, - // as defined by the 'All' description in the package comment. A return value of nil indicates no match. - FindAllIndex(data []byte, n int) [][]int - - // FindString returns a string holding the text of the leftmost match in s of the regular expression. - // If there is no match, the return value is an empty string, but it will also be empty - // if the regular expression successfully matches an empty string. - // Use FindStringIndex if it is necessary to distinguish these cases. - FindString(s string) string - - // FindStringIndex returns a two-element slice of integers defining - // the location of the leftmost match in s of the regular expression. - // The match itself is at s[loc[0]:loc[1]]. A return value of nil indicates no match. - FindStringIndex(s string) []int - - // FindAllString is the 'All' version of FindString; it returns a slice of all successive matches of the expression, - // as defined by the 'All' description in the package comment. A return value of nil indicates no match. - FindAllString(s string, n int) []string - - // FindAllStringIndex is the 'All' version of FindStringIndex; - // it returns a slice of all successive matches of the expression, - // as defined by the 'All' description in the package comment. A return value of nil indicates no match. - FindAllStringIndex(s string, n int) [][]int - - // Match reports whether the pattern database matches the byte slice b. - Match(b []byte) bool - - // MatchString reports whether the pattern database matches the string s. - MatchString(s string) bool -} - -// Stream exist in the Hyperscan library so that pattern matching state can be maintained -// across multiple blocks of target data. -type Stream interface { - Scan(data []byte) error - - Close() error - - Reset() error - - Clone() (Stream, error) -} - -// StreamScanner is the streaming regular expression scanner. -type StreamScanner interface { - Open(flags ScanFlag, scratch *Scratch, handler MatchHandler, context interface{}) (Stream, error) - - Scan(reader io.Reader, scratch *Scratch, handler MatchHandler, context interface{}) error -} - -// StreamMatcher implements regular expression search. -type StreamMatcher interface { - // Find returns a slice holding the text of the leftmost match in b of the regular expression. - // A return value of nil indicates no match. - Find(reader io.ReadSeeker) []byte - - // FindIndex returns a two-element slice of integers defining - // the location of the leftmost match in b of the regular expression. - // The match itself is at b[loc[0]:loc[1]]. A return value of nil indicates no match. - FindIndex(reader io.Reader) []int - - // FindAll is the 'All' version of Find; it returns a slice of all successive matches of the expression, - // as defined by the 'All' description in the package comment. A return value of nil indicates no match. - FindAll(reader io.ReadSeeker, n int) [][]byte - - // FindAllIndex is the 'All' version of FindIndex; it returns a slice of all successive matches of the expression, - // as defined by the 'All' description in the package comment. A return value of nil indicates no match. - FindAllIndex(reader io.Reader, n int) [][]int - - // Match reports whether the pattern database matches the byte slice b. - Match(reader io.Reader) bool -} - -// StreamCompressor implements stream compressor. -type StreamCompressor interface { - // Creates a compressed representation of the provided stream in the buffer provided. - Compress(stream Stream) ([]byte, error) - - // Decompresses a compressed representation created by `CompressStream` into a new stream. - Expand(buf []byte, flags ScanFlag, scratch *Scratch, handler MatchHandler, context interface{}) (Stream, error) - - // Decompresses a compressed representation created by `CompressStream` on top of the 'to' stream. - ResetAndExpand(stream Stream, buf []byte, flags ScanFlag, scratch *Scratch, - handler MatchHandler, context interface{}) (Stream, error) -} - -// VectoredScanner is the vectored regular expression scanner. -type VectoredScanner interface { - Scan(data [][]byte, scratch *Scratch, handler MatchHandler, context interface{}) error -} - -// VectoredMatcher implements regular expression search. -type VectoredMatcher interface{} - -type stream struct { - stream hsStream - flags ScanFlag - scratch hsScratch - handler hsMatchEventHandler - context interface{} - ownedScratch bool -} - -func (s *stream) Scan(data []byte) error { - return hsScanStream(s.stream, data, s.flags, s.scratch, s.handler, s.context) -} - -func (s *stream) Close() error { - err := hsCloseStream(s.stream, s.scratch, s.handler, s.context) - - if s.ownedScratch { - _ = hsFreeScratch(s.scratch) - } - - return err -} - -func (s *stream) Reset() error { - return hsResetStream(s.stream, s.flags, s.scratch, s.handler, s.context) -} - -func (s *stream) Clone() (Stream, error) { - ss, err := hsCopyStream(s.stream) - if err != nil { - return nil, err - } - - scratch := s.scratch - - if s.ownedScratch { - scratch, err = hsCloneScratch(s.scratch) - - if err != nil { - return nil, err - } - } - - return &stream{ss, s.flags, scratch, s.handler, s.context, s.ownedScratch}, nil -} - -type streamScanner struct { - *baseDatabase -} - -func newStreamScanner(db *baseDatabase) *streamScanner { - return &streamScanner{baseDatabase: db} -} - -func (ss *streamScanner) Open(flags ScanFlag, sc *Scratch, handler MatchHandler, context interface{}) (Stream, error) { - s, err := hsOpenStream(ss.db, flags) - if err != nil { - return nil, fmt.Errorf("open stream, %w", err) - } - - ownedScratch := false - - if sc == nil { - sc, err = NewScratch(ss) - if err != nil { - return nil, fmt.Errorf("create scratch, %w", err) - } - - ownedScratch = true - } - - return &stream{s, flags, sc.s, hsMatchEventHandler(handler), context, ownedScratch}, nil -} - -func (ss *streamScanner) Scan(reader io.Reader, scratch *Scratch, handler MatchHandler, context interface{}) error { - stream, err := ss.Open(0, nil, handler, context) - if err != nil { - return err - } - defer stream.Close() - - buf := make([]byte, bufSize) - - for { - n, err := reader.Read(buf) - - if err == io.EOF { - return nil - } - - if err != nil { - return fmt.Errorf("read stream, %w", err) - } - - if err = stream.Scan(buf[:n]); err != nil { - return err // nolint: wrapcheck - } - } -} - -type vectoredScanner struct { - *baseDatabase -} - -func newVectoredScanner(vdb *baseDatabase) *vectoredScanner { - return &vectoredScanner{vdb} -} - -func (vs *vectoredScanner) Scan(data [][]byte, s *Scratch, handler MatchHandler, context interface{}) (err error) { - if s == nil { - s, err = NewScratch(vs) - - if err != nil { - return - } - - defer func() { - _ = s.Free() - }() - } - - return hsScanVector(vs.db, data, 0, s.s, hsMatchEventHandler(handler), context) -} - -type blockScanner struct { - *baseDatabase -} - -func newBlockScanner(bdb *baseDatabase) *blockScanner { - return &blockScanner{bdb} -} - -func (bs *blockScanner) Scan(data []byte, s *Scratch, handler MatchHandler, context interface{}) (err error) { - if s == nil { - s, err = NewScratch(bs) - - if err != nil { - return - } - - defer func() { - _ = s.Free() - }() - } - - return hsScan(bs.db, data, 0, s.s, hsMatchEventHandler(handler), context) -} - -type blockMatcher struct { - *blockScanner - *matchRecorder - n int -} - -func newBlockMatcher(scanner *blockScanner) *blockMatcher { - return &blockMatcher{blockScanner: scanner} -} - -func (m *blockMatcher) Handle(id uint, from, to uint64, flags uint, context interface{}) error { - err := m.matchRecorder.Handle(id, from, to, flags, context) - if err != nil { - return err - } +type ScanFlag = hs.ScanFlag - if m.n < 0 { - return nil - } - - if m.n < len(m.matched) { - m.matched = m.matched[:m.n] - - return ErrTooManyMatches - } - - return nil -} - -func (m *blockMatcher) scan(data []byte) error { - m.matchRecorder = &matchRecorder{} - - return m.blockScanner.Scan(data, nil, m.Handle, nil) -} - -const findIndexMatches = 2 - -func (m *blockMatcher) Find(data []byte) []byte { - if loc := m.FindIndex(data); len(loc) == findIndexMatches { - return data[loc[0]:loc[1]] - } - - return nil -} - -func (m *blockMatcher) FindIndex(data []byte) []int { - if m.Match(data) && len(m.matched) == 1 { - return []int{int(m.matched[0].from), int(m.matched[0].to)} - } - - return nil -} - -func (m *blockMatcher) FindAll(data []byte, n int) (matches [][]byte) { - if locs := m.FindAllIndex(data, n); len(locs) > 0 { - for _, loc := range locs { - matches = append(matches, data[loc[0]:loc[1]]) - } - } - - return -} - -func (m *blockMatcher) FindAllIndex(data []byte, n int) (locs [][]int) { - if n < 0 { - n = len(data) + 1 - } - - m.n = n - - if err := m.scan(data); (err == nil || errors.Is(err, ErrScanTerminated)) && len(m.matched) > 0 { - for _, e := range m.matched { - locs = append(locs, []int{int(e.from), int(e.to)}) - } - } - - return -} - -func (m *blockMatcher) FindString(s string) string { - return string(m.Find([]byte(s))) -} - -func (m *blockMatcher) FindStringIndex(s string) (loc []int) { - return m.FindIndex([]byte(s)) -} - -func (m *blockMatcher) FindAllString(s string, n int) (results []string) { - for _, m := range m.FindAll([]byte(s), n) { - results = append(results, string(m)) - } - - return -} - -func (m *blockMatcher) FindAllStringIndex(s string, n int) [][]int { - return m.FindAllIndex([]byte(s), n) -} - -func (m *blockMatcher) Match(data []byte) bool { - m.n = 1 - - err := m.scan(data) - - return (err == nil || errors.Is(err, ErrScanTerminated)) && len(m.matched) == m.n -} - -func (m *blockMatcher) MatchString(s string) bool { - return m.Match([]byte(s)) -} - -type streamMatcher struct { - *streamScanner - *matchRecorder - n int -} - -func newStreamMatcher(scanner *streamScanner) *streamMatcher { - return &streamMatcher{streamScanner: scanner} -} - -func (m *streamMatcher) Handle(id uint, from, to uint64, flags uint, context interface{}) error { - err := m.matchRecorder.Handle(id, from, to, flags, context) - if err != nil { - return err - } - - if m.n < 0 { - return nil - } - - if m.n < len(m.matched) { - m.matched = m.matched[:m.n] - - return ErrTooManyMatches - } - - return nil -} - -func (m *streamMatcher) scan(reader io.Reader) error { - m.matchRecorder = &matchRecorder{} - - stream, err := m.streamScanner.Open(0, nil, m.Handle, nil) - if err != nil { - return err - } - defer stream.Close() - - buf := make([]byte, bufSize) - - for { - n, err := reader.Read(buf) - - if err == io.EOF { - return nil - } - - if err != nil { - return fmt.Errorf("read stream, %w", err) - } - - if err = stream.Scan(buf[:n]); err != nil { - return err // nolint: wrapcheck - } - } -} - -func (m *streamMatcher) read(reader io.ReadSeeker, loc []int) ([]byte, error) { - if len(loc) != findIndexMatches { - return nil, fmt.Errorf("location, %w", ErrInvalid) - } - - offset := int64(loc[0]) - size := loc[1] - loc[0] - - _, err := reader.Seek(offset, io.SeekStart) - if err != nil { - return nil, fmt.Errorf("seek stream, %w", err) - } - - buf := make([]byte, size) - - _, err = reader.Read(buf) - - if err != nil { - return nil, fmt.Errorf("read data, %w", err) - } - - return buf, nil -} - -func (m *streamMatcher) Find(reader io.ReadSeeker) []byte { - loc := m.FindIndex(reader) - - buf, err := m.read(reader, loc) - if err != nil { - return nil - } - - return buf -} - -func (m *streamMatcher) FindIndex(reader io.Reader) []int { - if m.Match(reader) && len(m.matched) == 1 { - return []int{int(m.matched[0].from), int(m.matched[0].to)} - } - - return nil -} - -func (m *streamMatcher) FindAll(reader io.ReadSeeker, n int) (result [][]byte) { - for _, loc := range m.FindAllIndex(reader, n) { - if buf, err := m.read(reader, loc); err == nil { - result = append(result, buf) - } - } - - return -} - -func (m *streamMatcher) FindAllIndex(reader io.Reader, n int) (locs [][]int) { - m.n = n - - if err := m.scan(reader); (err == nil || errors.Is(err, ErrScanTerminated)) && len(m.matched) > 0 { - for _, e := range m.matched { - locs = append(locs, []int{int(e.from), int(e.to)}) - } - } - - return -} - -func (m *streamMatcher) Match(reader io.Reader) bool { - m.n = 1 - - err := m.scan(reader) - - return (err == nil || errors.Is(err, ErrScanTerminated)) && len(m.matched) == m.n -} - -type vectoredMatcher struct { - *vectoredScanner -} - -func newVectoredMatcher(scanner *vectoredScanner) *vectoredMatcher { - return &vectoredMatcher{vectoredScanner: scanner} -} - -var _ StreamCompressor = (*streamDatabase)(nil) - -func (db *streamDatabase) Compress(s Stream) ([]byte, error) { - size, err := db.StreamSize() - if err != nil { - return nil, fmt.Errorf("stream size, %w", err) - } - - buf := make([]byte, size) - - buf, err = hsCompressStream(s.(*stream).stream, buf) - - if err != nil { - return nil, fmt.Errorf("compress stream, %w", err) - } - - return buf, nil -} - -func (db *streamDatabase) Expand(buf []byte, flags ScanFlag, sc *Scratch, - handler MatchHandler, context interface{}) (Stream, error) { - var s hsStream - - err := hsExpandStream(db.db, &s, buf) - if err != nil { - return nil, fmt.Errorf("expand stream, %w", err) - } - - ownedScratch := false - - if sc == nil { - sc, err = NewScratch(db) - if err != nil { - return nil, fmt.Errorf("create scratch, %w", err) - } - - ownedScratch = true - } - - return &stream{s, flags, sc.s, hsMatchEventHandler(handler), context, ownedScratch}, nil -} - -func (db *streamDatabase) ResetAndExpand(s Stream, buf []byte, flags ScanFlag, sc *Scratch, - handler MatchHandler, context interface{}) (Stream, error) { - ss, ok := s.(*stream) - if !ok { - return nil, fmt.Errorf("stream %v, %w", s, ErrUnexpected) - } - - ownedScratch := false - - if sc == nil { - var err error - - sc, err = NewScratch(db) - if err != nil { - return nil, fmt.Errorf("create scratch, %w", err) - } - - ownedScratch = true - } - - err := hsResetAndExpandStream(ss.stream, buf, ss.scratch, ss.handler, ss.context) - if err != nil { - return nil, fmt.Errorf("reset and expand stream, %w", err) - } - - return &stream{ss.stream, flags, sc.s, hsMatchEventHandler(handler), context, ownedScratch}, nil -} +// MatchHandler handles match events. +type MatchHandler = hs.MatchEventHandler diff --git a/hyperscan/scratch.go b/hyperscan/scratch.go new file mode 100644 index 0000000..9293828 --- /dev/null +++ b/hyperscan/scratch.go @@ -0,0 +1,60 @@ +package hyperscan + +import ( + "runtime" + + "github.com/flier/gohs/internal/hs" +) + +// Scratch is a Hyperscan scratch space. +type Scratch struct { + s hs.Scratch +} + +// NewScratch allocate a "scratch" space for use by Hyperscan. +// This is required for runtime use, and one scratch space per thread, +// or concurrent caller, is required. +func NewScratch(db Database) (*Scratch, error) { + s, err := hs.AllocScratch(db.(database).Db()) + if err != nil { + return nil, err // nolint: wrapcheck + } + + return &Scratch{s}, nil +} + +// NewManagedScratch is a wrapper for NewScratch that sets +// a finalizer on the Scratch instance so that memory is freed +// once the object is no longer in use. +func NewManagedScratch(db Database) (*Scratch, error) { + s, err := NewScratch(db) + if err != nil { + return nil, err + } + + runtime.SetFinalizer(s, func(scratch *Scratch) { + _ = scratch.Free() + }) + return s, nil +} + +// Size provides the size of the given scratch space. +func (s *Scratch) Size() (int, error) { return hs.ScratchSize(s.s) } // nolint: wrapcheck + +// Realloc reallocate the scratch for another database. +func (s *Scratch) Realloc(db Database) error { + return hs.ReallocScratch(db.(database).Db(), &s.s) // nolint: wrapcheck +} + +// Clone allocate a scratch space that is a clone of an existing scratch space. +func (s *Scratch) Clone() (*Scratch, error) { + cloned, err := hs.CloneScratch(s.s) + if err != nil { + return nil, err // nolint: wrapcheck + } + + return &Scratch{cloned}, nil +} + +// Free a scratch block previously allocated. +func (s *Scratch) Free() error { return hs.FreeScratch(s.s) } // nolint: wrapcheck diff --git a/hyperscan/stream.go b/hyperscan/stream.go new file mode 100644 index 0000000..ccb4c96 --- /dev/null +++ b/hyperscan/stream.go @@ -0,0 +1,368 @@ +package hyperscan + +import ( + "errors" + "fmt" + "io" + + "github.com/flier/gohs/internal/hs" +) + +// Stream exist in the Hyperscan library so that pattern matching state can be maintained +// across multiple blocks of target data. +type Stream interface { + Scan(data []byte) error + + Close() error + + Reset() error + + Clone() (Stream, error) +} + +// StreamScanner is the streaming regular expression scanner. +type StreamScanner interface { + Open(flags ScanFlag, scratch *Scratch, handler MatchHandler, context interface{}) (Stream, error) + + Scan(reader io.Reader, scratch *Scratch, handler MatchHandler, context interface{}) error +} + +// StreamMatcher implements regular expression search. +type StreamMatcher interface { + // Find returns a slice holding the text of the leftmost match in b of the regular expression. + // A return value of nil indicates no match. + Find(reader io.ReadSeeker) []byte + + // FindIndex returns a two-element slice of integers defining + // the location of the leftmost match in b of the regular expression. + // The match itself is at b[loc[0]:loc[1]]. A return value of nil indicates no match. + FindIndex(reader io.Reader) []int + + // FindAll is the 'All' version of Find; it returns a slice of all successive matches of the expression, + // as defined by the 'All' description in the package comment. A return value of nil indicates no match. + FindAll(reader io.ReadSeeker, n int) [][]byte + + // FindAllIndex is the 'All' version of FindIndex; it returns a slice of all successive matches of the expression, + // as defined by the 'All' description in the package comment. A return value of nil indicates no match. + FindAllIndex(reader io.Reader, n int) [][]int + + // Match reports whether the pattern database matches the byte slice b. + Match(reader io.Reader) bool +} + +// StreamCompressor implements stream compressor. +type StreamCompressor interface { + // Creates a compressed representation of the provided stream in the buffer provided. + Compress(stream Stream) ([]byte, error) + + // Decompresses a compressed representation created by `CompressStream` into a new stream. + Expand(buf []byte, flags ScanFlag, scratch *Scratch, handler MatchHandler, context interface{}) (Stream, error) + + // Decompresses a compressed representation created by `CompressStream` on top of the 'to' stream. + ResetAndExpand(stream Stream, buf []byte, flags ScanFlag, scratch *Scratch, + handler MatchHandler, context interface{}) (Stream, error) +} + +const bufSize = 4096 + +type stream struct { + stream hs.Stream + flags ScanFlag + scratch hs.Scratch + handler hs.MatchEventHandler + context interface{} + ownedScratch bool +} + +func (s *stream) Scan(data []byte) error { + return hs.ScanStream(s.stream, data, s.flags, s.scratch, s.handler, s.context) // nolint: wrapcheck +} + +func (s *stream) Close() error { + err := hs.CloseStream(s.stream, s.scratch, s.handler, s.context) + + if s.ownedScratch { + _ = hs.FreeScratch(s.scratch) + } + + return err // nolint: wrapcheck +} + +func (s *stream) Reset() error { + return hs.ResetStream(s.stream, s.flags, s.scratch, s.handler, s.context) // nolint: wrapcheck +} + +func (s *stream) Clone() (Stream, error) { + ss, err := hs.CopyStream(s.stream) + if err != nil { + return nil, fmt.Errorf("copy stream, %w", err) + } + + scratch := s.scratch + + if s.ownedScratch { + scratch, err = hs.CloneScratch(s.scratch) + + if err != nil { + hs.FreeStream(ss) + + return nil, fmt.Errorf("clone scratch, %w", err) + } + } + + return &stream{ss, s.flags, scratch, s.handler, s.context, s.ownedScratch}, nil +} + +type streamScanner struct { + *baseDatabase +} + +func newStreamScanner(db *baseDatabase) *streamScanner { + return &streamScanner{baseDatabase: db} +} + +func (ss *streamScanner) Open(flags ScanFlag, sc *Scratch, handler MatchHandler, context interface{}) (Stream, error) { + s, err := hs.OpenStream(ss.db, flags) + if err != nil { + return nil, fmt.Errorf("open stream, %w", err) + } + + ownedScratch := false + + if sc == nil { + sc, err = NewScratch(ss) + if err != nil { + return nil, fmt.Errorf("create scratch, %w", err) + } + + ownedScratch = true + } + + return &stream{s, flags, sc.s, handler, context, ownedScratch}, nil +} + +func (ss *streamScanner) Scan(reader io.Reader, scratch *Scratch, handler MatchHandler, context interface{}) error { + stream, err := ss.Open(0, nil, handler, context) + if err != nil { + return err + } + defer stream.Close() + + buf := make([]byte, bufSize) + + for { + n, err := reader.Read(buf) + + if err == io.EOF { + return nil + } + + if err != nil { + return fmt.Errorf("read stream, %w", err) + } + + if err = stream.Scan(buf[:n]); err != nil { + return err // nolint: wrapcheck + } + } +} + +type streamMatcher struct { + *streamScanner + *hs.MatchRecorder + n int +} + +func newStreamMatcher(scanner *streamScanner) *streamMatcher { + return &streamMatcher{streamScanner: scanner} +} + +func (m *streamMatcher) Handle(id uint, from, to uint64, flags uint, context interface{}) error { + err := m.MatchRecorder.Handle(id, from, to, flags, context) + if err != nil { + return err // nolint: wrapcheck + } + + if m.n < 0 { + return nil + } + + if m.n < len(m.Events) { + m.Events = m.Events[:m.n] + + return ErrTooManyMatches + } + + return nil +} + +func (m *streamMatcher) scan(reader io.Reader) error { + m.MatchRecorder = &hs.MatchRecorder{} + + stream, err := m.streamScanner.Open(0, nil, m.Handle, nil) + if err != nil { + return err + } + defer stream.Close() + + buf := make([]byte, bufSize) + + for { + n, err := reader.Read(buf) + + if err == io.EOF { + return nil + } + + if err != nil { + return fmt.Errorf("read stream, %w", err) + } + + if err = stream.Scan(buf[:n]); err != nil { + return err // nolint: wrapcheck + } + } +} + +func (m *streamMatcher) read(reader io.ReadSeeker, loc []int) ([]byte, error) { + if len(loc) != findIndexMatches { + return nil, fmt.Errorf("location, %w", ErrInvalid) + } + + offset := int64(loc[0]) + size := loc[1] - loc[0] + + _, err := reader.Seek(offset, io.SeekStart) + if err != nil { + return nil, fmt.Errorf("seek stream, %w", err) + } + + buf := make([]byte, size) + + _, err = reader.Read(buf) + + if err != nil { + return nil, fmt.Errorf("read data, %w", err) + } + + return buf, nil +} + +func (m *streamMatcher) Find(reader io.ReadSeeker) []byte { + loc := m.FindIndex(reader) + + buf, err := m.read(reader, loc) + if err != nil { + return nil + } + + return buf +} + +func (m *streamMatcher) FindIndex(reader io.Reader) []int { + if m.Match(reader) && len(m.Events) == 1 { + return []int{int(m.Events[0].From), int(m.Events[0].To)} + } + + return nil +} + +func (m *streamMatcher) FindAll(reader io.ReadSeeker, n int) (result [][]byte) { + for _, loc := range m.FindAllIndex(reader, n) { + if buf, err := m.read(reader, loc); err == nil { + result = append(result, buf) + } + } + + return +} + +func (m *streamMatcher) FindAllIndex(reader io.Reader, n int) (locs [][]int) { + m.n = n + + if err := m.scan(reader); (err == nil || errors.Is(err, ErrScanTerminated)) && len(m.Events) > 0 { + for _, e := range m.Events { + locs = append(locs, []int{int(e.From), int(e.To)}) + } + } + + return +} + +func (m *streamMatcher) Match(reader io.Reader) bool { + m.n = 1 + + err := m.scan(reader) + + return (err == nil || errors.Is(err, ErrScanTerminated)) && len(m.Events) == m.n +} + +var _ StreamCompressor = (*streamDatabase)(nil) + +func (db *streamDatabase) Compress(s Stream) ([]byte, error) { + size, err := db.StreamSize() + if err != nil { + return nil, fmt.Errorf("stream size, %w", err) + } + + buf := make([]byte, size) + + buf, err = hs.CompressStream(s.(*stream).stream, buf) + + if err != nil { + return nil, fmt.Errorf("compress stream, %w", err) + } + + return buf, nil +} + +func (db *streamDatabase) Expand(buf []byte, flags ScanFlag, sc *Scratch, + handler MatchHandler, context interface{}) (Stream, error) { + var s hs.Stream + + err := hs.ExpandStream(db.db, &s, buf) + if err != nil { + return nil, fmt.Errorf("expand stream, %w", err) + } + + ownedScratch := false + + if sc == nil { + sc, err = NewScratch(db) + if err != nil { + return nil, fmt.Errorf("create scratch, %w", err) + } + + ownedScratch = true + } + + return &stream{s, flags, sc.s, handler, context, ownedScratch}, nil +} + +func (db *streamDatabase) ResetAndExpand(s Stream, buf []byte, flags ScanFlag, sc *Scratch, + handler MatchHandler, context interface{}) (Stream, error) { + ss, ok := s.(*stream) + if !ok { + return nil, fmt.Errorf("stream %v, %w", s, ErrInvalid) + } + + ownedScratch := false + + if sc == nil { + var err error + + sc, err = NewScratch(db) + if err != nil { + return nil, fmt.Errorf("create scratch, %w", err) + } + + ownedScratch = true + } + + err := hs.ResetAndExpandStream(ss.stream, buf, ss.scratch, ss.handler, ss.context) + if err != nil { + return nil, fmt.Errorf("reset and expand stream, %w", err) + } + + return &stream{ss.stream, flags, sc.s, handler, context, ownedScratch}, nil +} diff --git a/hyperscan/vectored.go b/hyperscan/vectored.go new file mode 100644 index 0000000..e5f6453 --- /dev/null +++ b/hyperscan/vectored.go @@ -0,0 +1,43 @@ +package hyperscan + +import "github.com/flier/gohs/internal/hs" + +// VectoredScanner is the vectored regular expression scanner. +type VectoredScanner interface { + Scan(data [][]byte, scratch *Scratch, handler MatchHandler, context interface{}) error +} + +// VectoredMatcher implements regular expression search. +type VectoredMatcher interface{} + +type vectoredScanner struct { + *baseDatabase +} + +func newVectoredScanner(vdb *baseDatabase) *vectoredScanner { + return &vectoredScanner{vdb} +} + +func (vs *vectoredScanner) Scan(data [][]byte, s *Scratch, handler MatchHandler, context interface{}) (err error) { + if s == nil { + s, err = NewScratch(vs) + + if err != nil { + return + } + + defer func() { + _ = s.Free() + }() + } + + return hs.ScanVector(vs.db, data, 0, s.s, handler, context) // nolint: wrapcheck +} + +type vectoredMatcher struct { + *vectoredScanner +} + +func newVectoredMatcher(scanner *vectoredScanner) *vectoredMatcher { + return &vectoredMatcher{vectoredScanner: scanner} +} diff --git a/internal/ch/allocator.go b/internal/ch/allocator.go new file mode 100644 index 0000000..928d5f4 --- /dev/null +++ b/internal/ch/allocator.go @@ -0,0 +1,208 @@ +package ch + +import "unsafe" + +/* +#include +#include + +#include + +static inline void* aligned64_malloc(size_t size) { + void* result; +#ifdef _WIN32 + result = _aligned_malloc(size, 64); +#else + if (posix_memalign(&result, 64, size)) { + result = 0; + } +#endif + return result; +} + +static inline void aligned64_free(void *ptr) { +#ifdef _WIN32 + _aligned_free(ptr); +#else + free(ptr); +#endif +} + +extern void *chDefaultAlloc(size_t size); +extern void chDefaultFree(void *ptr); +extern void *chDbAlloc(size_t size); +extern void chDbFree(void *ptr); +extern void *chMiscAlloc(size_t size); +extern void chMiscFree(void *ptr); +extern void *chScratchAlloc(size_t size); +extern void chScratchFree(void *ptr); +*/ +import "C" // nolint: typecheck + +type ( + // The type of the callback function that will be used by Chimera to allocate more memory at runtime as required. + AllocFunc func(uint) unsafe.Pointer + // The type of the callback function that will be used by Chimera to free memory regions previously + // allocated using the @ref ch_alloc_t function. + FreeFunc func(unsafe.Pointer) +) + +type Allocator struct { + Alloc AllocFunc + Free FreeFunc +} + +func DefaultAllocator() *Allocator { return &allocator } +func DatabaseAllocator() *Allocator { return &dbAllocator } +func MiscAllocator() *Allocator { return &miscAllocator } +func ScratchAllocator() *Allocator { return &scratchAllocator } + +var ( + defaultAllocator = Allocator{DefaultAlloc, DefaultFree} + allocator = defaultAllocator + dbAllocator = defaultAllocator + miscAllocator = defaultAllocator + scratchAllocator = defaultAllocator +) + +func DefaultAlloc(size uint) unsafe.Pointer { + return C.malloc(C.size_t(size)) +} + +func DefaultFree(ptr unsafe.Pointer) { + C.free(ptr) +} + +func AlignedAlloc(size uint) unsafe.Pointer { + return C.aligned64_malloc(C.size_t(size)) +} + +func AlignedFree(ptr unsafe.Pointer) { + C.aligned64_free(ptr) +} + +//export chDefaultAlloc +func chDefaultAlloc(size C.size_t) unsafe.Pointer { + return allocator.Alloc(uint(size)) +} + +//export chDefaultFree +func chDefaultFree(ptr unsafe.Pointer) { + allocator.Free(ptr) +} + +// Set the allocate and free functions used by Chimera for allocating +// memory at runtime for stream state, scratch space, database bytecode, +// and various other data structure returned by the Chimera API. +func SetAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { + var ret C.ch_error_t + + if allocFunc == nil || freeFunc == nil { + allocator = defaultAllocator + ret = C.ch_set_allocator(nil, nil) + } else { + allocator = Allocator{allocFunc, freeFunc} + ret = C.ch_set_allocator(C.ch_alloc_t(C.chDefaultAlloc), C.ch_free_t(C.chDefaultFree)) + } + + if ret != C.CH_SUCCESS { + err = Error(ret) + } + + dbAllocator = allocator + miscAllocator = allocator + scratchAllocator = allocator + + return +} + +//export chDbAlloc +func chDbAlloc(size C.size_t) unsafe.Pointer { + return dbAllocator.Alloc(uint(size)) +} + +//export chDbFree +func chDbFree(ptr unsafe.Pointer) { + dbAllocator.Free(ptr) +} + +// Set the allocate and free functions used by Chimera for allocating memory +// for database bytecode produced by the compile calls (@ref ch_compile() and +// @ref ch_compile_multi()). +func SetDatabaseAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { + var ret C.ch_error_t + + if allocFunc == nil || freeFunc == nil { + dbAllocator = defaultAllocator + ret = C.ch_set_database_allocator(nil, nil) + } else { + dbAllocator = Allocator{allocFunc, freeFunc} + ret = C.ch_set_database_allocator(C.ch_alloc_t(C.chDefaultAlloc), C.ch_free_t(C.chDefaultFree)) + } + + if ret != C.CH_SUCCESS { + err = Error(ret) + } + + return +} + +//export chMiscAlloc +func chMiscAlloc(size C.size_t) unsafe.Pointer { + return miscAllocator.Alloc(uint(size)) +} + +//export chMiscFree +func chMiscFree(ptr unsafe.Pointer) { + miscAllocator.Free(ptr) +} + +// Set the allocate and free functions used by Chimera for allocating memory +// for items returned by the Chimera API such as @ref ch_compile_error_t. +func SetMiscAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { + var ret C.ch_error_t + + if allocFunc == nil || freeFunc == nil { + miscAllocator = defaultAllocator + ret = C.ch_set_misc_allocator(nil, nil) + } else { + miscAllocator = Allocator{allocFunc, freeFunc} + ret = C.ch_set_misc_allocator(C.ch_alloc_t(C.chDefaultAlloc), C.ch_free_t(C.chDefaultFree)) + } + + if ret != C.CH_SUCCESS { + err = Error(ret) + } + + return +} + +//export chScratchAlloc +func chScratchAlloc(size C.size_t) unsafe.Pointer { + return scratchAllocator.Alloc(uint(size)) +} + +//export chScratchFree +func chScratchFree(ptr unsafe.Pointer) { + scratchAllocator.Free(ptr) +} + +// Set the allocate and free functions used by Chimera for allocating memory +// for scratch space by @ref ch_alloc_scratch() and @ref ch_clone_scratch(). +func SetscratchAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { + var ret C.ch_error_t + + if allocFunc == nil || freeFunc == nil { + scratchAllocator = defaultAllocator + ret = C.ch_set_scratch_allocator(nil, nil) + } else { + scratchAllocator = Allocator{allocFunc, freeFunc} + ret = C.ch_set_scratch_allocator(C.ch_alloc_t(C.chDefaultAlloc), C.ch_free_t(C.chDefaultFree)) + } + + if ret != C.CH_SUCCESS { + err = Error(ret) + } + + return +} diff --git a/internal/ch/common.go b/internal/ch/common.go new file mode 100644 index 0000000..4b65106 --- /dev/null +++ b/internal/ch/common.go @@ -0,0 +1,45 @@ +package ch + +// #include +import "C" + +import "unsafe" + +type Database *C.ch_database_t + +func FreeDatabase(db Database) (err error) { + if ret := C.ch_free_database(db); ret != C.CH_SUCCESS { + err = Error(ret) + } + + return +} + +func Version() string { + return C.GoString(C.ch_version()) +} + +func DatabaseSize(db Database) (n int, err error) { + var size C.size_t + + if ret := C.ch_database_size(db, &size); ret != C.CH_SUCCESS { + err = Error(ret) + } else { + n = int(size) + } + + return +} + +func DatabaseInfo(db Database) (s string, err error) { + var info *C.char + + if ret := C.ch_database_info(db, &info); ret != C.HS_SUCCESS { + err = Error(ret) + } + + s = C.GoString(info) + C.free(unsafe.Pointer(info)) + + return +} diff --git a/internal/ch/common_test.go b/internal/ch/common_test.go new file mode 100644 index 0000000..d55d8a7 --- /dev/null +++ b/internal/ch/common_test.go @@ -0,0 +1,21 @@ +package ch + +import ( + "regexp" + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestVersion(t *testing.T) { + Convey("Given a Chimera version", t, func() { + ver := Version() + + So(ver, ShouldNotBeEmpty) + + matched, err := regexp.MatchString(`^\d\.\d\.\d.*`, ver) + + So(err, ShouldBeNil) + So(matched, ShouldBeTrue) + }) +} diff --git a/internal/ch/compile.go b/internal/ch/compile.go new file mode 100644 index 0000000..1e296a9 --- /dev/null +++ b/internal/ch/compile.go @@ -0,0 +1,186 @@ +package ch + +import ( + "fmt" + "runtime" + "sort" + "strings" + "unsafe" + + "github.com/flier/gohs/internal/hs" +) + +// #include +import "C" + +// Expression of pattern. +type Expression string + +func (e Expression) String() string { return string(e) } + +// Patterns is a set of matching patterns. +type Patterns []*Pattern + +// Pattern is a matching pattern. +// nolint: golint,revive,stylecheck +type Pattern struct { + Expression // The expression to parse. + Flags CompileFlag // Flags which modify the behaviour of the expression. + Id int // The ID number to be associated with the corresponding pattern +} + +func (p *Pattern) String() string { + var b strings.Builder + + if p.Id > 0 { + fmt.Fprintf(&b, "%d:", p.Id) + } + + fmt.Fprintf(&b, "/%s/%s", p.Expression, p.Flags) + + return b.String() +} + +// A type containing error details that is returned by the compile calls on failure. +// +// The caller may inspect the values returned in this type to determine the cause of failure. +type CompileError struct { + Message string // A human-readable error message describing the error. + Expression int // The zero-based number of the expression that caused the error. +} + +func (e *CompileError) Error() string { return e.Message } + +type CompileFlag uint + +const ( + // Caseless represents set case-insensitive matching. + Caseless CompileFlag = C.CH_FLAG_CASELESS + // DotAll represents matching a `.` will not exclude newlines. + DotAll CompileFlag = C.CH_FLAG_DOTALL + // MultiLine set multi-line anchoring. + MultiLine CompileFlag = C.CH_FLAG_MULTILINE + // SingleMatch set single-match only mode. + SingleMatch CompileFlag = C.CH_FLAG_SINGLEMATCH + // Utf8Mode enable UTF-8 mode for this expression. + Utf8Mode CompileFlag = C.CH_FLAG_UTF8 + // UnicodeProperty enable Unicode property support for this expression. + UnicodeProperty CompileFlag = C.HS_FLAG_UCP +) + +var CompileFlags = map[rune]CompileFlag{ + 'i': Caseless, + 's': DotAll, + 'm': MultiLine, + 'H': SingleMatch, + '8': Utf8Mode, + 'W': UnicodeProperty, +} + +func (flags CompileFlag) String() string { + var values []string + + for c, flag := range CompileFlags { + if (flags & flag) == flag { + values = append(values, string(c)) + } + } + + sort.Strings(values) + + return strings.Join(values, "") +} + +// CompileMode flags. +type CompileMode int + +const ( + // Disable capturing groups. + NoGroups CompileMode = C.CH_MODE_NOGROUPS + + // Enable capturing groups. + Groups CompileMode = C.CH_MODE_GROUPS +) + +// The basic regular expression compiler. +func Compile(expression string, flags CompileFlag, mode CompileMode, info *hs.PlatformInfo) (Database, error) { + var db *C.ch_database_t + var err *C.ch_compile_error_t + var platform *C.hs_platform_info_t + + if info != nil { + platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) + } + + expr := C.CString(expression) + + defer C.free(unsafe.Pointer(expr)) + + ret := C.ch_compile(expr, C.uint(flags), C.uint(mode), platform, &db, &err) + + if err != nil { + defer C.ch_free_compile_error(err) + } + + if ret == C.CH_SUCCESS { + return db, nil + } + + if ret == C.CH_COMPILER_ERROR && err != nil { + return nil, &CompileError{C.GoString(err.message), int(err.expression)} + } + + return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) +} + +// The multiple regular expression compiler. +func CompileMulti(patterns []*Pattern, mode CompileMode, info *hs.PlatformInfo) (Database, error) { + var db *C.ch_database_t + var err *C.ch_compile_error_t + var platform *C.hs_platform_info_t + + if info != nil { + platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) + } + + cexprs := (**C.char)(C.calloc(C.size_t(len(patterns)), C.size_t(unsafe.Sizeof(uintptr(0))))) + exprs := (*[1 << 30]*C.char)(unsafe.Pointer(cexprs))[:len(patterns):len(patterns)] + + cflags := (*C.uint)(C.calloc(C.size_t(len(patterns)), C.size_t(unsafe.Sizeof(C.uint(0))))) + flags := (*[1 << 30]C.uint)(unsafe.Pointer(cflags))[:len(patterns):len(patterns)] + + cids := (*C.uint)(C.calloc(C.size_t(len(patterns)), C.size_t(unsafe.Sizeof(C.uint(0))))) + ids := (*[1 << 30]C.uint)(unsafe.Pointer(cids))[:len(patterns):len(patterns)] + + for i, pattern := range patterns { + exprs[i] = C.CString(string(pattern.Expression)) + flags[i] = C.uint(pattern.Flags) + ids[i] = C.uint(pattern.Id) + } + + ret := C.ch_compile_multi(cexprs, cflags, cids, C.uint(len(patterns)), C.uint(mode), platform, &db, &err) + + for _, expr := range exprs { + C.free(unsafe.Pointer(expr)) + } + + C.free(unsafe.Pointer(cexprs)) + C.free(unsafe.Pointer(cflags)) + C.free(unsafe.Pointer(cids)) + + runtime.KeepAlive(patterns) + + if err != nil { + defer C.ch_free_compile_error(err) + } + + if ret == C.CH_SUCCESS { + return db, nil + } + + if ret == C.CH_COMPILER_ERROR && err != nil { + return nil, &CompileError{C.GoString(err.message), int(err.expression)} + } + + return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) +} diff --git a/chimera/dummy.cxx b/internal/ch/dummy.cxx similarity index 100% rename from chimera/dummy.cxx rename to internal/ch/dummy.cxx diff --git a/chimera/internal.go b/internal/ch/error.go similarity index 65% rename from chimera/internal.go rename to internal/ch/error.go index 53fd22b..d127f1c 100644 --- a/chimera/internal.go +++ b/internal/ch/error.go @@ -1,50 +1,41 @@ -package chimera +package ch -/* -#cgo pkg-config: libch libhs -#cgo linux LDFLAGS: -lm -lpcre -#cgo darwin LDFLAGS: -lpcre - -#include -#include -#include - -#include -*/ +// #include import "C" + import "fmt" -// ChError represents an error -type ChError C.ch_error_t +// Error represents an error. +type Error C.ch_error_t const ( // ErrSuccess is the error returned if the engine completed normally. - ErrSuccess ChError = C.CH_SUCCESS + ErrSuccess Error = C.CH_SUCCESS // ErrInvalid is the error returned if a parameter passed to this function was invalid. - ErrInvalid ChError = C.CH_INVALID + ErrInvalid Error = C.CH_INVALID // ErrNoMemory is the error returned if a memory allocation failed. - ErrNoMemory ChError = C.CH_NOMEM + ErrNoMemory Error = C.CH_NOMEM // ErrScanTerminated is the error returned if the engine was terminated by callback. - ErrScanTerminated ChError = C.CH_SCAN_TERMINATED + ErrScanTerminated Error = C.CH_SCAN_TERMINATED // ErrCompileError is the error returned if the pattern compiler failed. - ErrCompileError ChError = C.CH_COMPILER_ERROR + ErrCompileError Error = C.CH_COMPILER_ERROR // ErrDatabaseVersionError is the error returned if the given database was built for a different version of Hyperscan. - ErrDatabaseVersionError ChError = C.CH_DB_VERSION_ERROR + ErrDatabaseVersionError Error = C.CH_DB_VERSION_ERROR // ErrDatabasePlatformError is the error returned if the given database was built for a different platform (i.e., CPU type). - ErrDatabasePlatformError ChError = C.CH_DB_PLATFORM_ERROR + ErrDatabasePlatformError Error = C.CH_DB_PLATFORM_ERROR // ErrDatabaseModeError is the error returned if the given database was built for a different mode of operation. - ErrDatabaseModeError ChError = C.CH_DB_MODE_ERROR + ErrDatabaseModeError Error = C.CH_DB_MODE_ERROR // ErrBadAlign is the error returned if a parameter passed to this function was not correctly aligned. - ErrBadAlign ChError = C.CH_BAD_ALIGN + ErrBadAlign Error = C.CH_BAD_ALIGN // ErrBadAlloc is the error returned if the memory allocator did not correctly return memory suitably aligned. - ErrBadAlloc ChError = C.CH_BAD_ALLOC + ErrBadAlloc Error = C.CH_BAD_ALLOC // ErrScratchInUse is the error returned if the scratch region was already in use. - ErrScratchInUse ChError = C.CH_SCRATCH_IN_USE + ErrScratchInUse Error = C.CH_SCRATCH_IN_USE // ErrUnknown is the unexpected internal error from Hyperscan. - ErrUnknownHSError ChError = C.CH_UNKNOWN_HS_ERROR + ErrUnknownHSError Error = C.CH_UNKNOWN_HS_ERROR ) -var chErrorMessages = map[ChError]string{ +var ErrorMessages = map[Error]string{ C.CH_SUCCESS: "The engine completed normally.", C.CH_INVALID: "A parameter passed to this function was invalid.", C.CH_NOMEM: "A memory allocation failed.", @@ -59,19 +50,10 @@ var chErrorMessages = map[ChError]string{ C.CH_UNKNOWN_HS_ERROR: "Unexpected internal error from Hyperscan.", } -func (e ChError) Error() string { - if msg, exists := chErrorMessages[e]; exists { +func (e Error) Error() string { + if msg, exists := ErrorMessages[e]; exists { return msg } - return fmt.Sprintf("unexpected error, %d", int(e)) -} - -type ( - chDatabase *C.ch_database_t - chScratch *C.ch_scratch_t -) - -func chVersion() string { - return C.GoString(C.ch_version()) + return fmt.Sprintf("unexpected error, %d", int(C.ch_error_t(e))) } diff --git a/internal/ch/link.go b/internal/ch/link.go new file mode 100644 index 0000000..9d3e6ee --- /dev/null +++ b/internal/ch/link.go @@ -0,0 +1,8 @@ +package ch + +/* +#cgo pkg-config: libch libhs +#cgo linux LDFLAGS: -lm -lpcre +#cgo darwin LDFLAGS: -lpcre +*/ +import "C" diff --git a/internal/ch/runtime.go b/internal/ch/runtime.go new file mode 100644 index 0000000..8e4c390 --- /dev/null +++ b/internal/ch/runtime.go @@ -0,0 +1,160 @@ +package ch + +import ( + "reflect" + "runtime" + "unsafe" + + "github.com/flier/gohs/internal/handle" +) + +/* +#include + +typedef const ch_capture_t capture_t; + +extern ch_callback_t matchEventCallback(unsigned int id, + unsigned long long from, + unsigned long long to, + unsigned int flags, + unsigned int size, + const ch_capture_t *captured, + void *ctx); + +extern ch_callback_t errorEventCallback(ch_error_event_t error_type, + unsigned int id, + void *info, + void *ctx); +*/ +import "C" + +// A Chimera scratch space. +type Scratch *C.ch_scratch_t + +// Allocate a scratch space that is a clone of an existing scratch space. +func CloneScratch(scratch Scratch) (Scratch, error) { + var clone *C.ch_scratch_t + + if ret := C.ch_clone_scratch(scratch, &clone); ret != C.HS_SUCCESS { + return nil, Error(ret) + } + + return clone, nil +} + +// Provides the size of the given scratch space. +func ScratchSize(scratch Scratch) (int, error) { + var size C.size_t + + if ret := C.ch_scratch_size(scratch, &size); ret != C.HS_SUCCESS { + return 0, Error(ret) + } + + return int(size), nil +} + +// Free a scratch block previously allocated by @ref ch_alloc_scratch() or @ref ch_clone_scratch(). +func FreeScratch(scratch Scratch) error { + if ret := C.ch_free_scratch(scratch); ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +// Callback return value used to tell the Chimera matcher what to do after processing this match. +type Callback C.ch_callback_t + +const ( + Continue Callback = C.CH_CALLBACK_CONTINUE // Continue matching. + Terminate Callback = C.CH_CALLBACK_TERMINATE // Terminate matching. + SkipPattern Callback = C.CH_CALLBACK_SKIP_PATTERN // Skip remaining matches for this ID and continue. +) + +// Capture representing a captured subexpression within a match. +type Capture struct { + From uint64 // offset at which this capture group begins. + To uint64 // offset at which this capture group ends. +} + +// Definition of the match event callback function type. +type MatchEventHandler func(id uint, from, to uint64, flags uint, captured []*Capture, context interface{}) Callback + +type ErrorEvent C.ch_error_event_t + +const ( + // PCRE hits its match limit and reports PCRE_ERROR_MATCHLIMIT. + MatchLimit ErrorEvent = C.CH_ERROR_MATCHLIMIT + // PCRE hits its recursion limit and reports PCRE_ERROR_RECURSIONLIMIT. + RecursionLimit ErrorEvent = C.CH_ERROR_RECURSIONLIMIT +) + +// Definition of the Chimera error event callback function type. +type ErrorEventHandler func(event ErrorEvent, id uint, info, context interface{}) Callback + +type eventContext struct { + onEvent MatchEventHandler + onError ErrorEventHandler + context interface{} +} + +//export matchEventCallback +func matchEventCallback(id C.uint, from, to C.ulonglong, flags, size C.uint, cap *C.capture_t, data unsafe.Pointer) C.ch_callback_t { + ctx, ok := handle.Handle(data).Value().(eventContext) + if !ok { + return C.CH_CALLBACK_TERMINATE + } + + captured := make([]*Capture, size) + for i, c := range (*[1 << 30]C.capture_t)(unsafe.Pointer(cap))[:size:size] { + if c.flags == C.CH_CAPTURE_FLAG_ACTIVE { + captured[i] = &Capture{uint64(c.from), uint64(c.to)} + } + } + + return C.ch_callback_t(ctx.onEvent(uint(id), uint64(from), uint64(to), uint(flags), captured, ctx.context)) +} + +//export errorEventCallback +func errorEventCallback(evt C.ch_error_event_t, id C.uint, info, data unsafe.Pointer) C.ch_callback_t { + ctx, ok := handle.Handle(data).Value().(eventContext) + if !ok { + return C.CH_CALLBACK_TERMINATE + } + + return C.ch_callback_t(ctx.onError(ErrorEvent(evt), uint(id), nil, ctx.context)) +} + +// ScanFlag represents a scan flag. +type ScanFlag C.uint + +// The block regular expression scanner. +func hsScan(db Database, data []byte, flags ScanFlag, scratch Scratch, + onEvent MatchEventHandler, onError ErrorEventHandler, context interface{}) error { + if data == nil { + return ErrInvalid + } + + h := handle.New(eventContext{onEvent, onError, context}) + defer h.Delete() + + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&data)) // FIXME: Zero-copy access to go data + + ret := C.ch_scan(db, + (*C.char)(unsafe.Pointer(hdr.Data)), + C.uint(hdr.Len), + C.uint(flags), + scratch, + C.ch_match_event_handler(C.matchEventCallback), + C.ch_error_event_handler(C.errorEventCallback), + unsafe.Pointer(h)) + + // Ensure go data is alive before the C function returns + runtime.KeepAlive(data) + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} diff --git a/hyperscan/handle/go1_15.go b/internal/handle/go1_15.go similarity index 100% rename from hyperscan/handle/go1_15.go rename to internal/handle/go1_15.go diff --git a/hyperscan/handle/go1_17.go b/internal/handle/go1_17.go similarity index 100% rename from hyperscan/handle/go1_17.go rename to internal/handle/go1_17.go diff --git a/hyperscan/handle/go1_9.go b/internal/handle/go1_9.go similarity index 100% rename from hyperscan/handle/go1_9.go rename to internal/handle/go1_9.go diff --git a/hyperscan/handle/handle.go b/internal/handle/handle.go similarity index 100% rename from hyperscan/handle/handle.go rename to internal/handle/handle.go diff --git a/internal/hs/allocator.go b/internal/hs/allocator.go new file mode 100644 index 0000000..3081692 --- /dev/null +++ b/internal/hs/allocator.go @@ -0,0 +1,248 @@ +package hs + +import "unsafe" + +/* +#include +#include + +#include + +static inline void* aligned64_malloc(size_t size) { + void* result; +#ifdef _WIN32 + result = _aligned_malloc(size, 64); +#else + if (posix_memalign(&result, 64, size)) { + result = 0; + } +#endif + return result; +} + +static inline void aligned64_free(void *ptr) { +#ifdef _WIN32 + _aligned_free(ptr); +#else + free(ptr); +#endif +} + +extern void *hsAlloc(size_t size); +extern void hsFree(void *ptr); +extern void *hsDbAlloc(size_t size); +extern void hsDbFree(void *ptr); +extern void *hsMiscAlloc(size_t size); +extern void hsMiscFree(void *ptr); +extern void *hsScratchAlloc(size_t size); +extern void hsScratchFree(void *ptr); +extern void *hsStreamAlloc(size_t size); +extern void hsStreamFree(void *ptr); +*/ +import "C" + +type ( + AllocFunc func(uint) unsafe.Pointer + FreeFunc func(unsafe.Pointer) +) + +type Allocator struct { + Alloc AllocFunc + Free FreeFunc +} + +func DefaultAllocator() *Allocator { return &allocator } +func DatabaseAllocator() *Allocator { return &dbAllocator } +func MiscAllocator() *Allocator { return &miscAllocator } +func ScratchAllocator() *Allocator { return &scratchAllocator } +func StreamAllocator() *Allocator { return &streamAllocator } + +var ( + defaultAllocator = Allocator{DefaultAlloc, DefaultFree} + allocator = defaultAllocator + dbAllocator = defaultAllocator + miscAllocator = defaultAllocator + scratchAllocator = defaultAllocator + streamAllocator = defaultAllocator +) + +func DefaultAlloc(size uint) unsafe.Pointer { + return C.malloc(C.size_t(size)) +} + +func DefaultFree(ptr unsafe.Pointer) { + C.free(ptr) +} + +func AlignedAlloc(size uint) unsafe.Pointer { + return C.aligned64_malloc(C.size_t(size)) +} + +func AlignedFree(ptr unsafe.Pointer) { + C.aligned64_free(ptr) +} + +//export hsAlloc +func hsAlloc(size C.size_t) unsafe.Pointer { + return allocator.Alloc(uint(size)) +} + +//export hsFree +func hsFree(ptr unsafe.Pointer) { + allocator.Free(ptr) +} + +func SetAllocator(allocFunc AllocFunc, freeFunc FreeFunc) error { + var ret C.hs_error_t + + if allocFunc == nil || freeFunc == nil { + allocator = defaultAllocator + ret = C.hs_set_allocator(nil, nil) + } else { + allocator = Allocator{allocFunc, freeFunc} + ret = C.hs_set_allocator(C.hs_alloc_t(C.hsAlloc), C.hs_free_t(C.hsFree)) + } + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + dbAllocator = allocator + miscAllocator = allocator + scratchAllocator = allocator + streamAllocator = allocator + + return nil +} + +func ClearAllocator() error { + return SetAllocator(nil, nil) +} + +//export hsDbAlloc +func hsDbAlloc(size C.size_t) unsafe.Pointer { + return dbAllocator.Alloc(uint(size)) +} + +//export hsDbFree +func hsDbFree(ptr unsafe.Pointer) { + dbAllocator.Free(ptr) +} + +func SetDatabaseAllocator(allocFunc AllocFunc, freeFunc FreeFunc) error { + var ret C.hs_error_t + + if allocFunc == nil || freeFunc == nil { + dbAllocator = defaultAllocator + ret = C.hs_set_database_allocator(nil, nil) + } else { + dbAllocator = Allocator{allocFunc, freeFunc} + ret = C.hs_set_database_allocator(C.hs_alloc_t(C.hsDbAlloc), C.hs_free_t(C.hsDbFree)) + } + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func ClearDatabaseAllocator() error { + return SetDatabaseAllocator(nil, nil) +} + +//export hsMiscAlloc +func hsMiscAlloc(size C.size_t) unsafe.Pointer { + return miscAllocator.Alloc(uint(size)) +} + +//export hsMiscFree +func hsMiscFree(ptr unsafe.Pointer) { + miscAllocator.Free(ptr) +} + +func SetMiscAllocator(allocFunc AllocFunc, freeFunc FreeFunc) error { + var ret C.hs_error_t + + if allocFunc == nil || freeFunc == nil { + miscAllocator = defaultAllocator + ret = C.hs_set_misc_allocator(nil, nil) + } else { + miscAllocator = Allocator{allocFunc, freeFunc} + ret = C.hs_set_misc_allocator(C.hs_alloc_t(C.hsMiscAlloc), C.hs_free_t(C.hsMiscFree)) + } + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func ClearMiscAllocator() error { + return SetMiscAllocator(nil, nil) +} + +//export hsScratchAlloc +func hsScratchAlloc(size C.size_t) unsafe.Pointer { + return scratchAllocator.Alloc(uint(size)) +} + +//export hsScratchFree +func hsScratchFree(ptr unsafe.Pointer) { + scratchAllocator.Free(ptr) +} + +func SetScratchAllocator(allocFunc AllocFunc, freeFunc FreeFunc) error { + var ret C.hs_error_t + + if allocFunc == nil || freeFunc == nil { + scratchAllocator = defaultAllocator + ret = C.hs_set_scratch_allocator(nil, nil) + } else { + scratchAllocator = Allocator{allocFunc, freeFunc} + ret = C.hs_set_scratch_allocator(C.hs_alloc_t(C.hsScratchAlloc), C.hs_free_t(C.hsScratchFree)) + } + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func ClearScratchAllocator() error { + return SetScratchAllocator(nil, nil) +} + +//export hsStreamAlloc +func hsStreamAlloc(size C.size_t) unsafe.Pointer { + return streamAllocator.Alloc(uint(size)) +} + +//export hsStreamFree +func hsStreamFree(ptr unsafe.Pointer) { + streamAllocator.Free(ptr) +} + +func SetStreamAllocator(allocFunc AllocFunc, freeFunc FreeFunc) error { + var ret C.hs_error_t + + if allocFunc == nil || freeFunc == nil { + streamAllocator = defaultAllocator + ret = C.hs_set_stream_allocator(nil, nil) + } else { + streamAllocator = Allocator{allocFunc, freeFunc} + ret = C.hs_set_stream_allocator(C.hs_alloc_t(C.hsStreamAlloc), C.hs_free_t(C.hsStreamFree)) + } + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func ClearStreamAllocator() error { + return SetStreamAllocator(nil, nil) +} diff --git a/internal/hs/allocator_test.go b/internal/hs/allocator_test.go new file mode 100644 index 0000000..1ae8f4f --- /dev/null +++ b/internal/hs/allocator_test.go @@ -0,0 +1,138 @@ +package hs_test + +import ( + "testing" + "unsafe" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/internal/hs" +) + +type testAllocator struct { + memoryUsed int + memoryFreed []unsafe.Pointer +} + +func (a *testAllocator) alloc(size uint) unsafe.Pointer { + a.memoryUsed += int(size) + + return hs.AlignedAlloc(size) +} + +func (a *testAllocator) free(ptr unsafe.Pointer) { + a.memoryFreed = append(a.memoryFreed, ptr) + + hs.AlignedFree(ptr) +} + +//nolint:funlen +func TestAllocator(t *testing.T) { + Convey("Given the host platform", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + a := &testAllocator{} + + Convey("Given a simple expression with allocator", func() { + So(hs.SetMiscAllocator(a.alloc, a.free), ShouldBeNil) + + info, err := hs.ExpressionInfo("test", 0) + + So(info, ShouldNotBeNil) + So(info, ShouldResemble, &hs.ExprInfo{ + MinWidth: 4, + MaxWidth: 4, + }) + So(err, ShouldBeNil) + + So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, 12) + + So(hs.ClearMiscAllocator(), ShouldBeNil) + }) + + Convey("Then create a stream database with allocator", func() { + So(hs.SetDatabaseAllocator(a.alloc, a.free), ShouldBeNil) + + db, err := hs.Compile("test", 0, hs.StreamMode, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the database size", func() { + size, err := hs.DatabaseSize(db) + + So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, size) + So(err, ShouldBeNil) + }) + + Convey("Then create a scratch with allocator", func() { + So(hs.SetScratchAllocator(a.alloc, a.free), ShouldBeNil) + + a.memoryUsed = 0 + + s, err := hs.AllocScratch(db) + + So(s, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the scratch size", func() { + size, err := hs.ScratchSize(s) + + So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, size) + So(err, ShouldBeNil) + }) + + Convey("Then open a stream", func() { + So(hs.SetStreamAllocator(a.alloc, a.free), ShouldBeNil) + + a.memoryUsed = 0 + + stream, err := hs.OpenStream(db, 0) + + So(stream, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the stream size", func() { + size, err := hs.StreamSize(db) + + So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, size) + So(err, ShouldBeNil) + }) + + h := &hs.MatchRecorder{} + + Convey("Then close stream with allocator", func() { + a.memoryFreed = nil + + So(hs.CloseStream(stream, s, h.Handle, nil), ShouldBeNil) + + So(hs.ClearStreamAllocator(), ShouldBeNil) + }) + }) + + Convey("Then free scratch with allocator", func() { + a.memoryFreed = nil + + So(hs.FreeScratch(s), ShouldBeNil) + + So(a.memoryFreed, ShouldResemble, []unsafe.Pointer{unsafe.Pointer(s)}) + + So(hs.ClearScratchAllocator(), ShouldBeNil) + }) + }) + + Convey("Then free database with allocator", func() { + a.memoryFreed = nil + + So(hs.FreeDatabase(db), ShouldBeNil) + + So(a.memoryFreed, ShouldResemble, []unsafe.Pointer{unsafe.Pointer(db)}) + + So(hs.ClearDatabaseAllocator(), ShouldBeNil) + }) + }) + }) +} diff --git a/internal/hs/common.go b/internal/hs/common.go new file mode 100644 index 0000000..ff5cd8a --- /dev/null +++ b/internal/hs/common.go @@ -0,0 +1,135 @@ +package hs + +import ( + "runtime" + "unsafe" +) + +// #include +import "C" + +type Database *C.hs_database_t + +func Version() string { + return C.GoString(C.hs_version()) +} + +func ValidPlatform() error { + if ret := C.hs_valid_platform(); ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func FreeDatabase(db Database) (err error) { + if ret := C.hs_free_database(db); ret != C.HS_SUCCESS { + err = Error(ret) + } + + return +} + +func SerializeDatabase(db Database) (b []byte, err error) { + var data *C.char + var length C.size_t + + ret := C.hs_serialize_database(db, &data, &length) + if ret != C.HS_SUCCESS { + err = Error(ret) + } else { + defer C.free(unsafe.Pointer(data)) + + b = C.GoBytes(unsafe.Pointer(data), C.int(length)) + } + + return +} + +func DeserializeDatabase(data []byte) (Database, error) { + var db *C.hs_database_t + + ret := C.hs_deserialize_database((*C.char)(unsafe.Pointer(&data[0])), C.size_t(len(data)), &db) + + runtime.KeepAlive(data) + + if ret != C.HS_SUCCESS { + return nil, Error(ret) + } + + return db, nil +} + +func DeserializeDatabaseAt(data []byte, db Database) error { + ret := C.hs_deserialize_database_at((*C.char)(unsafe.Pointer(&data[0])), C.size_t(len(data)), db) + + runtime.KeepAlive(data) + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func StreamSize(db Database) (int, error) { + var size C.size_t + + if ret := C.hs_stream_size(db, &size); ret != C.HS_SUCCESS { + return 0, Error(ret) + } + + return int(size), nil +} + +func DatabaseSize(db Database) (int, error) { + var size C.size_t + + if ret := C.hs_database_size(db, &size); ret != C.HS_SUCCESS { + return -1, Error(ret) + } + + return int(size), nil +} + +func SerializedDatabaseSize(data []byte) (int, error) { + var size C.size_t + + ret := C.hs_serialized_database_size((*C.char)(unsafe.Pointer(&data[0])), C.size_t(len(data)), &size) + + runtime.KeepAlive(data) + + if ret != C.HS_SUCCESS { + return 0, Error(ret) + } + + return int(size), nil +} + +func DatabaseInfo(db Database) (string, error) { + var info *C.char + + if ret := C.hs_database_info(db, &info); ret != C.HS_SUCCESS { + return "", Error(ret) + } + + defer C.free(unsafe.Pointer(info)) + + return C.GoString(info), nil +} + +func SerializedDatabaseInfo(data []byte) (string, error) { + var info *C.char + + ret := C.hs_serialized_database_info((*C.char)(unsafe.Pointer(&data[0])), C.size_t(len(data)), &info) + + runtime.KeepAlive(data) + + if ret != C.HS_SUCCESS { + return "", Error(ret) + } + + defer C.free(unsafe.Pointer(info)) + + return C.GoString(info), nil +} diff --git a/internal/hs/common_test.go b/internal/hs/common_test.go new file mode 100644 index 0000000..eb5ed31 --- /dev/null +++ b/internal/hs/common_test.go @@ -0,0 +1,126 @@ +package hs_test + +import ( + "regexp" + "testing" + "unsafe" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/internal/hs" +) + +func TestVersion(t *testing.T) { + Convey("Given a HyperScan version", t, func() { + ver := hs.Version() + + So(ver, ShouldNotBeEmpty) + + matched, err := regexp.MatchString(`^\d\.\d\.\d.*`, ver) + + So(err, ShouldBeNil) + So(matched, ShouldBeTrue) + }) +} + +var regexInfo = regexp.MustCompile(`^Version: (\d+\.\d+\.\d+) Features: ([\w\s]+)? Mode: (\w+)$`) + +//nolint:funlen +func TestDatabase(t *testing.T) { + Convey("Given a stream database", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + db, err := hs.Compile("test", 0, hs.StreamMode, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the database info", func() { + info, err := hs.DatabaseInfo(db) + + So(regexInfo.MatchString(info), ShouldBeTrue) + So(err, ShouldBeNil) + }) + + Convey("Get the database size", func() { + size, err := hs.DatabaseSize(db) + + So(size, ShouldBeGreaterThan, 800) + So(err, ShouldBeNil) + }) + + Convey("Get the stream size", func() { + size, err := hs.StreamSize(db) + + So(size, ShouldBeGreaterThan, 20) + So(err, ShouldBeNil) + }) + + Convey("Get the stream size from a block database", func() { + db, err := hs.Compile("test", 0, hs.BlockMode, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + size, err := hs.StreamSize(db) + + So(size, ShouldEqual, 0) + So(err, ShouldEqual, hs.ErrDatabaseModeError) + }) + + Convey("When serialize database", func() { + data, err := hs.SerializeDatabase(db) + + So(data, ShouldNotBeNil) + So(len(data), ShouldBeGreaterThan, 800) + So(err, ShouldBeNil) + + Convey("Get the database info", func() { + info, err := hs.SerializedDatabaseInfo(data) + + So(regexInfo.MatchString(info), ShouldBeTrue) + So(err, ShouldBeNil) + }) + + Convey("Get the database size", func() { + size, err := hs.SerializedDatabaseSize(data) + + So(size, ShouldBeGreaterThan, 800) + So(err, ShouldBeNil) + }) + + Convey("Then deserialize database", func() { + db, err := hs.DeserializeDatabase(data) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the database info", func() { + info, err := hs.DatabaseInfo(db) + + So(regexInfo.MatchString(info), ShouldBeTrue) + So(err, ShouldBeNil) + }) + }) + + Convey("Then deserialize database to memory", func() { + buf := make([]byte, 1000) + db := hs.Database(unsafe.Pointer(&buf[0])) + + So(hs.DeserializeDatabaseAt(data, db), ShouldBeNil) + + Convey("Get the database info", func() { + info, err := hs.DatabaseInfo(db) + + So(regexInfo.MatchString(info), ShouldBeTrue) + So(err, ShouldBeNil) + }) + }) + }) + + So(hs.FreeDatabase(db), ShouldBeNil) + }) +} diff --git a/internal/hs/compile.go b/internal/hs/compile.go new file mode 100644 index 0000000..31d3be8 --- /dev/null +++ b/internal/hs/compile.go @@ -0,0 +1,434 @@ +package hs + +/* +#include + +#include +*/ +import "C" + +import ( + "errors" + "fmt" + "runtime" + "sort" + "strings" + "unsafe" +) + +var ( + // ErrNoFound means patterns not found. + ErrNoFound = errors.New("no found") + // ErrUnexpected means item is unexpected. + ErrUnexpected = errors.New("unexpected") +) + +// A type containing error details that is returned by the compile calls on failure. +// +// The caller may inspect the values returned in this type to determine the cause of failure. +type CompileError struct { + Message string // A human-readable error message describing the error. + Expression int // The zero-based number of the expression that caused the error. +} + +func (e *CompileError) Error() string { return e.Message } + +// CpuFeature is the CPU feature support flags +type CpuFeature int // nolint: golint,stylecheck + +const ( + // AVX2 is a CPU features flag indicates that the target platform supports AVX2 instructions. + AVX2 CpuFeature = C.HS_CPU_FEATURES_AVX2 + // AVX512 is a CPU features flag indicates that the target platform supports AVX512 instructions, specifically AVX-512BW. Using AVX512 implies the use of AVX2. + AVX512 CpuFeature = C.HS_CPU_FEATURES_AVX512 +) + +// TuneFlag is the tuning flags +type TuneFlag int + +const ( + // Generic indicates that the compiled database should not be tuned for any particular target platform. + Generic TuneFlag = C.HS_TUNE_FAMILY_GENERIC + // SandyBridge indicates that the compiled database should be tuned for the Sandy Bridge microarchitecture. + SandyBridge TuneFlag = C.HS_TUNE_FAMILY_SNB + // IvyBridge indicates that the compiled database should be tuned for the Ivy Bridge microarchitecture. + IvyBridge TuneFlag = C.HS_TUNE_FAMILY_IVB + // Haswell indicates that the compiled database should be tuned for the Haswell microarchitecture. + Haswell TuneFlag = C.HS_TUNE_FAMILY_HSW + // Silvermont indicates that the compiled database should be tuned for the Silvermont microarchitecture. + Silvermont TuneFlag = C.HS_TUNE_FAMILY_SLM + // Broadwell indicates that the compiled database should be tuned for the Broadwell microarchitecture. + Broadwell TuneFlag = C.HS_TUNE_FAMILY_BDW + // Skylake indicates that the compiled database should be tuned for the Skylake microarchitecture. + Skylake TuneFlag = C.HS_TUNE_FAMILY_SKL + // SkylakeServer indicates that the compiled database should be tuned for the Skylake Server microarchitecture. + SkylakeServer TuneFlag = C.HS_TUNE_FAMILY_SKX + // Goldmont indicates that the compiled database should be tuned for the Goldmont microarchitecture. + Goldmont TuneFlag = C.HS_TUNE_FAMILY_GLM +) + +type PlatformInfo struct { + Platform C.struct_hs_platform_info +} + +// Tune returns the tuning flags of the platform. +func (i *PlatformInfo) Tune() TuneFlag { return TuneFlag(i.Platform.tune) } + +// CpuFeatures returns the CPU features of the platform. +func (i *PlatformInfo) CpuFeatures() CpuFeature { return CpuFeature(i.Platform.cpu_features) } // nolint: golint,stylecheck + +func NewPlatformInfo(tune TuneFlag, cpu CpuFeature) *PlatformInfo { + var platform C.struct_hs_platform_info + + platform.tune = C.uint(tune) + platform.cpu_features = C.ulonglong(cpu) + + return &PlatformInfo{platform} +} + +func PopulatePlatform() (*PlatformInfo, error) { + var platform C.struct_hs_platform_info + + if ret := C.hs_populate_platform(&platform); ret != C.HS_SUCCESS { + return nil, Error(ret) + } + + return &PlatformInfo{platform}, nil +} + +// ExprInfo containing information related to an expression +type ExprInfo struct { + MinWidth uint // The minimum length in bytes of a match for the pattern. + MaxWidth uint // The maximum length in bytes of a match for the pattern. + ReturnUnordered bool // Whether this expression can produce matches that are not returned in order, such as those produced by assertions. + AtEndOfData bool // Whether this expression can produce matches at end of data (EOD). + OnlyAtEndOfData bool // Whether this expression can *only* produce matches at end of data (EOD). +} + +// UnboundedMaxWidth represents the pattern expression has an unbounded maximum width +const UnboundedMaxWidth = C.UINT_MAX + +func NewExprInfo(info *C.hs_expr_info_t) *ExprInfo { + return &ExprInfo{ + MinWidth: uint(info.min_width), + MaxWidth: uint(info.max_width), + ReturnUnordered: info.unordered_matches != 0, + AtEndOfData: info.matches_at_eod != 0, + OnlyAtEndOfData: info.matches_only_at_eod != 0, + } +} + +// ExtFlag are used in ExprExt.Flags to indicate which fields are used. +type ExtFlag uint64 + +const ( + // ExtMinOffset is a flag indicating that the ExprExt.MinOffset field is used. + ExtMinOffset ExtFlag = C.HS_EXT_FLAG_MIN_OFFSET + // ExtMaxOffset is a flag indicating that the ExprExt.MaxOffset field is used. + ExtMaxOffset ExtFlag = C.HS_EXT_FLAG_MAX_OFFSET + // ExtMinLength is a flag indicating that the ExprExt.MinLength field is used. + ExtMinLength ExtFlag = C.HS_EXT_FLAG_MIN_LENGTH + // ExtEditDistance is a flag indicating that the ExprExt.EditDistance field is used. + ExtEditDistance ExtFlag = C.HS_EXT_FLAG_EDIT_DISTANCE + // ExtHammingDistance is a flag indicating that the ExprExt.HammingDistance field is used. + ExtHammingDistance ExtFlag = C.HS_EXT_FLAG_HAMMING_DISTANCE +) + +// ExprExt is a structure containing additional parameters related to an expression. +type ExprExt struct { + Flags ExtFlag // Flags governing which parts of this structure are to be used by the compiler. + MinOffset uint64 // The minimum end offset in the data stream at which this expression should match successfully. + MaxOffset uint64 // The maximum end offset in the data stream at which this expression should match successfully. + MinLength uint64 // The minimum match length (from start to end) required to successfully match this expression. + EditDistance uint32 // Allow patterns to approximately match within this edit distance. + HammingDistance uint32 // Allow patterns to approximately match within this Hamming distance. +} + +func (e *ExprExt) c() *C.hs_expr_ext_t { + if e == nil { + return nil + } + + var ext C.hs_expr_ext_t + + if e.Flags&ExtMinOffset != 0 { + ext.flags |= C.HS_EXT_FLAG_MIN_OFFSET + ext.min_offset = C.ulonglong(e.MinOffset) + } + if e.Flags&ExtMaxOffset != 0 { + ext.flags |= C.HS_EXT_FLAG_MAX_OFFSET + ext.max_offset = C.ulonglong(e.MaxOffset) + } + if e.Flags&ExtMinLength != 0 { + ext.flags |= C.HS_EXT_FLAG_MIN_LENGTH + ext.min_length = C.ulonglong(e.MinLength) + } + if e.Flags&ExtEditDistance != 0 { + ext.flags |= C.HS_EXT_FLAG_EDIT_DISTANCE + ext.edit_distance = C.uint(e.EditDistance) + } + if e.Flags&ExtHammingDistance != 0 { + ext.flags |= C.HS_EXT_FLAG_HAMMING_DISTANCE + ext.hamming_distance = C.uint(e.HammingDistance) + } + + return &ext +} + +func ExpressionInfo(expression string, flags CompileFlag) (*ExprInfo, error) { + var info *C.hs_expr_info_t + var err *C.hs_compile_error_t + + expr := C.CString(expression) + + defer C.free(unsafe.Pointer(expr)) + + ret := C.hs_expression_info(expr, C.uint(flags), &info, &err) + + if ret == C.HS_SUCCESS && info != nil { + defer MiscAllocator().Free(unsafe.Pointer(info)) + + return NewExprInfo(info), nil + } + + if err != nil { + defer C.hs_free_compile_error(err) + } + + if ret == C.HS_COMPILER_ERROR && err != nil { + return nil, &CompileError{C.GoString(err.message), int(err.expression)} + } + + return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) +} + +func ExpressionExt(expression string, flags CompileFlag) (ext *ExprExt, info *ExprInfo, err error) { + var exprInfo *C.hs_expr_info_t + var compileErr *C.hs_compile_error_t + + ext = new(ExprExt) + expr := C.CString(expression) + + defer C.free(unsafe.Pointer(expr)) + + ret := C.hs_expression_ext_info(expr, C.uint(flags), (*C.hs_expr_ext_t)(unsafe.Pointer(ext)), &exprInfo, &compileErr) + + if exprInfo != nil { + defer MiscAllocator().Free(unsafe.Pointer(exprInfo)) + + info = NewExprInfo(exprInfo) + } + + if compileErr != nil { + defer C.hs_free_compile_error(compileErr) + } + + if ret == C.HS_COMPILER_ERROR && compileErr != nil { + err = &CompileError{C.GoString(compileErr.message), int(compileErr.expression)} + } else { + err = fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) + } + + return +} + +// CompileFlag represents a pattern flag +type CompileFlag uint + +const ( + // Caseless represents set case-insensitive matching. + Caseless CompileFlag = C.HS_FLAG_CASELESS + // DotAll represents matching a `.` will not exclude newlines. + DotAll CompileFlag = C.HS_FLAG_DOTALL + // MultiLine set multi-line anchoring. + MultiLine CompileFlag = C.HS_FLAG_MULTILINE + // SingleMatch set single-match only mode. + SingleMatch CompileFlag = C.HS_FLAG_SINGLEMATCH + // AllowEmpty allow expressions that can match against empty buffers. + AllowEmpty CompileFlag = C.HS_FLAG_ALLOWEMPTY + // Utf8Mode enable UTF-8 mode for this expression. + Utf8Mode CompileFlag = C.HS_FLAG_UTF8 + // UnicodeProperty enable Unicode property support for this expression. + UnicodeProperty CompileFlag = C.HS_FLAG_UCP + // PrefilterMode enable prefiltering mode for this expression. + PrefilterMode CompileFlag = C.HS_FLAG_PREFILTER + // SomLeftMost enable leftmost start of match reporting. + SomLeftMost CompileFlag = C.HS_FLAG_SOM_LEFTMOST +) + +var CompileFlags = map[rune]CompileFlag{ + 'i': Caseless, + 's': DotAll, + 'm': MultiLine, + 'H': SingleMatch, + 'V': AllowEmpty, + '8': Utf8Mode, + 'W': UnicodeProperty, + 'P': PrefilterMode, + 'L': SomLeftMost, +} + +var DeprecatedCompileFlags = map[rune]CompileFlag{ + 'o': SingleMatch, + 'e': AllowEmpty, + 'u': Utf8Mode, + 'p': UnicodeProperty, + 'f': PrefilterMode, + 'l': SomLeftMost, +} + +func (flags CompileFlag) String() string { + var values []string + + for c, flag := range CompileFlags { + if (flags & flag) == flag { + values = append(values, string(c)) + } + } + + sort.Strings(values) + + return strings.Join(values, "") +} + +// ModeFlag represents the compile mode flags +type ModeFlag uint + +const ( + // BlockMode for the block scan (non-streaming) database. + BlockMode ModeFlag = C.HS_MODE_BLOCK + // NoStreamMode is alias for Block. + NoStreamMode ModeFlag = C.HS_MODE_NOSTREAM + // StreamMode for the streaming database. + StreamMode ModeFlag = C.HS_MODE_STREAM + // VectoredMode for the vectored scanning database. + VectoredMode ModeFlag = C.HS_MODE_VECTORED + // SomHorizonLargeMode use full precision to track start of match offsets in stream state. + SomHorizonLargeMode ModeFlag = C.HS_MODE_SOM_HORIZON_LARGE + // SomHorizonMediumMode use medium precision to track start of match offsets in stream state. (within 2^32 bytes) + SomHorizonMediumMode ModeFlag = C.HS_MODE_SOM_HORIZON_MEDIUM + // SomHorizonSmallMode use limited precision to track start of match offsets in stream state. (within 2^16 bytes) + SomHorizonSmallMode ModeFlag = C.HS_MODE_SOM_HORIZON_SMALL + // ModeMask represents the mask of database mode + ModeMask ModeFlag = 0xFF +) + +var ModeFlags = map[string]ModeFlag{ + "STREAM": StreamMode, + "NOSTREAM": BlockMode, + "VECTORED": VectoredMode, + "BLOCK": BlockMode, +} + +func (m ModeFlag) String() string { + switch m & 0xF { + case BlockMode: + return "BLOCK" + case StreamMode: + return "STREAM" + case VectoredMode: + return "VECTORED" + default: + panic(fmt.Sprintf("unknown mode: %d", m)) + } +} + +func Compile(expression string, flags CompileFlag, mode ModeFlag, info *PlatformInfo) (Database, error) { + var db *C.hs_database_t + var err *C.hs_compile_error_t + var platform *C.hs_platform_info_t + + if info != nil { + platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) + } + + expr := C.CString(expression) + + defer C.free(unsafe.Pointer(expr)) + + ret := C.hs_compile(expr, C.uint(flags), C.uint(mode), platform, &db, &err) + + if err != nil { + defer C.hs_free_compile_error(err) + } + + if ret == C.HS_SUCCESS { + return db, nil + } + + if ret == C.HS_COMPILER_ERROR && err != nil { + return nil, &CompileError{C.GoString(err.message), int(err.expression)} + } + + return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) +} + +type Pattern struct { + Expr string + Flags CompileFlag + ID int + Ext *ExprExt +} + +type Patterns interface { + Patterns() []*Pattern +} + +func CompileMulti(input Patterns, mode ModeFlag, info *PlatformInfo) (Database, error) { + var db *C.hs_database_t + var err *C.hs_compile_error_t + var platform *C.hs_platform_info_t + + if info != nil { + platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) + } + + patterns := input.Patterns() + count := len(patterns) + + cexprs := (**C.char)(C.calloc(C.size_t(count), C.size_t(unsafe.Sizeof(uintptr(0))))) + exprs := (*[1 << 30]*C.char)(unsafe.Pointer(cexprs))[:count:count] + + cflags := (*C.uint)(C.calloc(C.size_t(count), C.size_t(unsafe.Sizeof(C.uint(0))))) + flags := (*[1 << 30]C.uint)(unsafe.Pointer(cflags))[:count:count] + + cids := (*C.uint)(C.calloc(C.size_t(count), C.size_t(unsafe.Sizeof(C.uint(0))))) + ids := (*[1 << 30]C.uint)(unsafe.Pointer(cids))[:count:count] + + cexts := (**C.hs_expr_ext_t)(C.calloc(C.size_t(count), C.size_t(unsafe.Sizeof(uintptr(0))))) + exts := (*[1 << 30]*C.hs_expr_ext_t)(unsafe.Pointer(cexts))[:count:count] + + for i, pattern := range patterns { + exprs[i] = C.CString(pattern.Expr) + flags[i] = C.uint(pattern.Flags) + ids[i] = C.uint(pattern.ID) + exts[i] = pattern.Ext.c() + } + + ret := C.hs_compile_ext_multi(cexprs, cflags, cids, cexts, C.uint(count), C.uint(mode), platform, &db, &err) + + for _, expr := range exprs { + C.free(unsafe.Pointer(expr)) + } + + C.free(unsafe.Pointer(cexprs)) + C.free(unsafe.Pointer(cflags)) + C.free(unsafe.Pointer(cexts)) + C.free(unsafe.Pointer(cids)) + + runtime.KeepAlive(patterns) + + if err != nil { + defer C.hs_free_compile_error(err) + } + + if ret == C.HS_SUCCESS { + return db, nil + } + + if ret == C.HS_COMPILER_ERROR && err != nil { + return nil, &CompileError{C.GoString(err.message), int(err.expression)} + } + + return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) +} diff --git a/internal/hs/compile_test.go b/internal/hs/compile_test.go new file mode 100644 index 0000000..1c70eb8 --- /dev/null +++ b/internal/hs/compile_test.go @@ -0,0 +1,135 @@ +package hs_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/hyperscan" + "github.com/flier/gohs/internal/hs" +) + +//nolint:funlen +func TestCompileAPI(t *testing.T) { + Convey("Given a host platform", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("When Compile a unsupported expression", func() { + Convey("Then compile as stream", func() { + db, err := hs.Compile(`\R`, 0, hs.StreamMode, platform) + + So(db, ShouldBeNil) + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, `\R at index 0 not supported.`) + }) + + Convey("Then compile as vector", func() { + db, err := hs.CompileMulti(hyperscan.NewPattern(`\R`, hs.Caseless), hs.BlockMode, platform) + + So(db, ShouldBeNil) + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, `\R at index 0 not supported.`) + }) + + Convey("Then compile as extended vector", func() { + db, err := hs.CompileMulti(hyperscan.NewPattern(`\R`, hs.Caseless, hyperscan.MinOffset(10)), + hs.VectoredMode, platform) + + So(db, ShouldBeNil) + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, `\R at index 0 not supported.`) + }) + }) + + Convey("Compile an empty expression", func() { + db, err := hs.Compile("", 0, hs.StreamMode, platform) + + So(db, ShouldBeNil) + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "Pattern matches empty buffer; use HS_FLAG_ALLOWEMPTY to enable support.") + + So(hs.FreeDatabase(db), ShouldBeNil) + }) + + Convey("Compile multi expressions", func() { + db, err := hs.CompileMulti(hyperscan.Patterns{ + hyperscan.NewPattern(`^\w+`, 0), + hyperscan.NewPattern(`\d+`, 0), + hyperscan.NewPattern(`\s+`, 0), + }, hs.StreamMode, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the database info", func() { + info, err := hs.DatabaseInfo(db) + + So(regexInfo.MatchString(info), ShouldBeTrue) + So(err, ShouldBeNil) + }) + + So(hs.FreeDatabase(db), ShouldBeNil) + }) + + Convey("Compile multi expressions with extension", func() { + db, err := hs.CompileMulti(hyperscan.Patterns{ + hyperscan.NewPattern(`^\w+`, 0, hyperscan.MinOffset(10)), + hyperscan.NewPattern(`\d+`, 0, hyperscan.MaxOffset(10)), + hyperscan.NewPattern(`\s+`, 0, hyperscan.MinLength(10)), + }, hs.StreamMode, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the database info", func() { + info, err := hs.DatabaseInfo(db) + + So(regexInfo.MatchString(info), ShouldBeTrue) + So(err, ShouldBeNil) + }) + + So(hs.FreeDatabase(db), ShouldBeNil) + }) + }) +} + +func TestExpression(t *testing.T) { + Convey("Given a simple expression", t, func() { + info, err := hs.ExpressionInfo("test", 0) + + So(info, ShouldNotBeNil) + So(info, ShouldResemble, &hs.ExprInfo{ + MinWidth: 4, + MaxWidth: 4, + }) + So(err, ShouldBeNil) + }) + + Convey("Given a credit card expression", t, func() { + info, err := hs.ExpressionInfo(hyperscan.CreditCard, 0) + + So(info, ShouldNotBeNil) + So(info, ShouldResemble, &hs.ExprInfo{ + MinWidth: 13, + MaxWidth: 16, + }) + So(err, ShouldBeNil) + }) + + Convey("Given a expression match eod", t, func() { + info, err := hs.ExpressionInfo("test$", 0) + + So(info, ShouldNotBeNil) + So(info, ShouldResemble, &hs.ExprInfo{ + MinWidth: 4, + MaxWidth: 4, + ReturnUnordered: true, + AtEndOfData: true, + OnlyAtEndOfData: true, + }) + So(err, ShouldBeNil) + }) +} diff --git a/hyperscan/internal_v5.go b/internal/hs/compile_v5.go similarity index 65% rename from hyperscan/internal_v5.go rename to internal/hs/compile_v5.go index 7b2ac2f..d7957ac 100644 --- a/hyperscan/internal_v5.go +++ b/internal/hs/compile_v5.go @@ -1,7 +1,7 @@ //go:build !hyperscan_v4 // +build !hyperscan_v4 -package hyperscan +package hs /* #include @@ -13,11 +13,6 @@ import ( "unsafe" ) -const ( - // ErrUnknown is an unexpected internal error. - ErrUnknown HsError = C.HS_UNKNOWN_ERROR -) - const ( // Combination represents logical combination. Combination CompileFlag = C.HS_FLAG_COMBINATION @@ -26,19 +21,28 @@ const ( ) func init() { - hsErrorMessages[C.HS_UNKNOWN_ERROR] = "Unexpected internal error." + CompileFlags['C'] = Combination + CompileFlags['Q'] = Quiet +} - compileFlags['C'] = Combination - compileFlags['Q'] = Quiet +// Pure literal is a special case of regular expression. +// A character sequence is regarded as a pure literal if and +// only if each character is read and interpreted independently. +// No syntax association happens between any adjacent characters. +type Literal struct { + Expr string // The expression to parse. + Flags CompileFlag // Flags which modify the behaviour of the expression. + ID int // The ID number to be associated with the corresponding pattern + *ExprInfo } -func hsCompileLit(expression string, flags CompileFlag, mode ModeFlag, info *hsPlatformInfo) (hsDatabase, error) { +func CompileLit(expression string, flags CompileFlag, mode ModeFlag, info *PlatformInfo) (Database, error) { var db *C.hs_database_t var err *C.hs_compile_error_t var platform *C.hs_platform_info_t if info != nil { - platform = &info.platform + platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) } expr := C.CString(expression) @@ -56,21 +60,26 @@ func hsCompileLit(expression string, flags CompileFlag, mode ModeFlag, info *hsP } if ret == C.HS_COMPILER_ERROR && err != nil { - return nil, &compileError{C.GoString(err.message), int(err.expression)} + return nil, &CompileError{C.GoString(err.message), int(err.expression)} } return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) } -func hsCompileLitMulti(literals []*Literal, mode ModeFlag, info *hsPlatformInfo) (hsDatabase, error) { +type Literals interface { + Literals() []*Literal +} + +func CompileLitMulti(input Literals, mode ModeFlag, info *PlatformInfo) (Database, error) { var db *C.hs_database_t var err *C.hs_compile_error_t var platform *C.hs_platform_info_t if info != nil { - platform = &info.platform + platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) } + literals := input.Literals() count := len(literals) cexprs := (**C.char)(C.calloc(C.size_t(len(literals)), C.size_t(unsafe.Sizeof(uintptr(0))))) @@ -86,10 +95,10 @@ func hsCompileLitMulti(literals []*Literal, mode ModeFlag, info *hsPlatformInfo) ids := (*[1 << 30]C.uint)(unsafe.Pointer(cids))[:len(literals):len(literals)] for i, lit := range literals { - exprs[i] = C.CString(string(lit.Expression)) - lens[i] = C.size_t(len(lit.Expression)) + exprs[i] = C.CString(lit.Expr) + lens[i] = C.size_t(len(lit.Expr)) flags[i] = C.uint(lit.Flags) - ids[i] = C.uint(lit.Id) + ids[i] = C.uint(lit.ID) } ret := C.hs_compile_lit_multi(cexprs, cflags, cids, clens, C.uint(count), C.uint(mode), platform, &db, &err) @@ -112,7 +121,7 @@ func hsCompileLitMulti(literals []*Literal, mode ModeFlag, info *hsPlatformInfo) } if ret == C.HS_COMPILER_ERROR && err != nil { - return nil, &compileError{C.GoString(err.message), int(err.expression)} + return nil, &CompileError{C.GoString(err.message), int(err.expression)} } return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) diff --git a/internal/hs/compile_v54.go b/internal/hs/compile_v54.go new file mode 100644 index 0000000..1af2268 --- /dev/null +++ b/internal/hs/compile_v54.go @@ -0,0 +1,22 @@ +//go:build hyperscan_v54 +// +build hyperscan_v54 + +package hs + +/* +#include +*/ +import "C" + +const ( + // AVX512VBMI is a CPU features flag indicates that the target platform + // supports Intel(R) Advanced Vector Extensions 512 Vector Byte Manipulation Instructions (Intel(R) AVX512VBMI) + AVX512VBMI CpuFeature = C.HS_CPU_FEATURES_AVX512VBMI +) + +const ( + // Icelake indicates that the compiled database should be tuned for the Icelake microarchitecture. + Icelake TuneFlag = C.HS_TUNE_FAMILY_ICL + // IcelakeServer indicates that the compiled database should be tuned for the Icelake Server microarchitecture. + IcelakeServer TuneFlag = C.HS_TUNE_FAMILY_ICX +) diff --git a/hyperscan/dummy.cxx b/internal/hs/dummy.cxx similarity index 100% rename from hyperscan/dummy.cxx rename to internal/hs/dummy.cxx diff --git a/internal/hs/error.go b/internal/hs/error.go new file mode 100644 index 0000000..8232e2a --- /dev/null +++ b/internal/hs/error.go @@ -0,0 +1,61 @@ +package hs + +// #include +import "C" +import "fmt" + +// Error represents an error +type Error int + +const ( + // ErrSuccess is the error returned if the engine completed normally. + ErrSuccess Error = C.HS_SUCCESS + // ErrInvalid is the error returned if a parameter passed to this function was invalid. + ErrInvalid Error = C.HS_INVALID + // ErrNoMemory is the error returned if a memory allocation failed. + ErrNoMemory Error = C.HS_NOMEM + // ErrScanTerminated is the error returned if the engine was terminated by callback. + ErrScanTerminated Error = C.HS_SCAN_TERMINATED + // ErrCompileError is the error returned if the pattern compiler failed. + ErrCompileError Error = C.HS_COMPILER_ERROR + // ErrDatabaseVersionError is the error returned if the given database was built for a different version of Hyperscan. + ErrDatabaseVersionError Error = C.HS_DB_VERSION_ERROR + // ErrDatabasePlatformError is the error returned if the given database was built for a different platform (i.e., CPU type). + ErrDatabasePlatformError Error = C.HS_DB_PLATFORM_ERROR + // ErrDatabaseModeError is the error returned if the given database was built for a different mode of operation. + ErrDatabaseModeError Error = C.HS_DB_MODE_ERROR + // ErrBadAlign is the error returned if a parameter passed to this function was not correctly aligned. + ErrBadAlign Error = C.HS_BAD_ALIGN + // ErrBadAlloc is the error returned if the memory allocator did not correctly return memory suitably aligned. + ErrBadAlloc Error = C.HS_BAD_ALLOC + // ErrScratchInUse is the error returned if the scratch region was already in use. + ErrScratchInUse Error = C.HS_SCRATCH_IN_USE + // ErrArchError is the error returned if unsupported CPU architecture. + ErrArchError Error = C.HS_ARCH_ERROR + // ErrInsufficientSpace is the error returned if provided buffer was too small. + ErrInsufficientSpace Error = C.HS_INSUFFICIENT_SPACE +) + +var errorMessages = map[Error]string{ + C.HS_SUCCESS: "The engine completed normally.", + C.HS_INVALID: "A parameter passed to this function was invalid.", + C.HS_NOMEM: "A memory allocation failed.", + C.HS_SCAN_TERMINATED: "The engine was terminated by callback.", + C.HS_COMPILER_ERROR: "The pattern compiler failed.", + C.HS_DB_VERSION_ERROR: "The given database was built for a different version of Hyperscan.", + C.HS_DB_PLATFORM_ERROR: "The given database was built for a different platform (i.e., CPU type).", + C.HS_DB_MODE_ERROR: "The given database was built for a different mode of operation.", + C.HS_BAD_ALIGN: "A parameter passed to this function was not correctly aligned.", + C.HS_BAD_ALLOC: "The memory allocator did not correctly return aligned memory.", + C.HS_SCRATCH_IN_USE: "The scratch region was already in use.", + C.HS_ARCH_ERROR: "Unsupported CPU architecture.", + C.HS_INSUFFICIENT_SPACE: "Provided buffer was too small.", +} + +func (e Error) Error() string { + if msg, exists := errorMessages[e]; exists { + return msg + } + + return fmt.Sprintf("unexpected error, %d", int(e)) +} diff --git a/internal/hs/error_v5.go b/internal/hs/error_v5.go new file mode 100644 index 0000000..76d80d5 --- /dev/null +++ b/internal/hs/error_v5.go @@ -0,0 +1,18 @@ +//go:build !hyperscan_v4 +// +build !hyperscan_v4 + +package hs + +/* +#include +*/ +import "C" + +const ( + // ErrUnknown is an unexpected internal error. + ErrUnknown Error = C.HS_UNKNOWN_ERROR +) + +func init() { + errorMessages[C.HS_UNKNOWN_ERROR] = "Unexpected internal error." +} diff --git a/internal/hs/link.go b/internal/hs/link.go new file mode 100644 index 0000000..b40e672 --- /dev/null +++ b/internal/hs/link.go @@ -0,0 +1,7 @@ +package hs + +/* +#cgo pkg-config: libhs +#cgo linux LDFLAGS: -lm +*/ +import "C" diff --git a/internal/hs/runtime.go b/internal/hs/runtime.go new file mode 100644 index 0000000..dcc6078 --- /dev/null +++ b/internal/hs/runtime.go @@ -0,0 +1,154 @@ +package hs + +import ( + "errors" + "reflect" + "runtime" + "unsafe" + + "github.com/flier/gohs/internal/handle" +) + +/* +#include + +extern int hsMatchEventCallback(unsigned int id, + unsigned long long from, + unsigned long long to, + unsigned int flags, + void *context); +*/ +import "C" + +// ScanFlag represents a scan flag +type ScanFlag uint + +type MatchEventHandler func(id uint, from, to uint64, flags uint, context interface{}) error + +type MatchEventContext struct { + handler MatchEventHandler + context interface{} +} + +//export hsMatchEventCallback +func hsMatchEventCallback(id C.uint, from, to C.ulonglong, flags C.uint, data unsafe.Pointer) C.int { + ctx, ok := handle.Handle(data).Value().(MatchEventContext) + if !ok { + return C.HS_INVALID + } + + err := ctx.handler(uint(id), uint64(from), uint64(to), uint(flags), ctx.context) + if err != nil { + var hsErr Error + if errors.As(err, &hsErr) { + return C.int(hsErr) + } + + return C.HS_SCAN_TERMINATED + } + + return C.HS_SUCCESS +} + +func Scan(db Database, data []byte, flags ScanFlag, scratch Scratch, onEvent MatchEventHandler, context interface{}) error { + if data == nil { + return Error(C.HS_INVALID) + } + + h := handle.New(MatchEventContext{onEvent, context}) + defer h.Delete() + + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&data)) // FIXME: Zero-copy access to go data + + ret := C.hs_scan(db, + (*C.char)(unsafe.Pointer(hdr.Data)), + C.uint(hdr.Len), + C.uint(flags), + scratch, + C.match_event_handler(C.hsMatchEventCallback), + unsafe.Pointer(h)) + + // Ensure go data is alive before the C function returns + runtime.KeepAlive(data) + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func ScanVector(db Database, data [][]byte, flags ScanFlag, scratch Scratch, onEvent MatchEventHandler, context interface{}) error { + if data == nil { + return Error(C.HS_INVALID) + } + + cdata := make([]uintptr, len(data)) + clength := make([]C.uint, len(data)) + + for i, d := range data { + if d == nil { + return Error(C.HS_INVALID) + } + + // FIXME: Zero-copy access to go data + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&d)) // nolint: scopelint + cdata[i] = uintptr(unsafe.Pointer(hdr.Data)) + clength[i] = C.uint(hdr.Len) + } + + h := handle.New(MatchEventContext{onEvent, context}) + defer h.Delete() + + cdataHdr := (*reflect.SliceHeader)(unsafe.Pointer(&cdata)) // FIXME: Zero-copy access to go data + clengthHdr := (*reflect.SliceHeader)(unsafe.Pointer(&clength)) // FIXME: Zero-copy access to go data + + ret := C.hs_scan_vector(db, + (**C.char)(unsafe.Pointer(cdataHdr.Data)), + (*C.uint)(unsafe.Pointer(clengthHdr.Data)), + C.uint(cdataHdr.Len), + C.uint(flags), + scratch, + C.match_event_handler(C.hsMatchEventCallback), + unsafe.Pointer(h)) + + // Ensure go data is alive before the C function returns + runtime.KeepAlive(data) + runtime.KeepAlive(cdata) + runtime.KeepAlive(clength) + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +type MatchEvent struct { + ID uint + From, To uint64 + ScanFlag +} + +type MatchRecorder struct { + Events []MatchEvent + Err error +} + +func (h *MatchRecorder) Matched() bool { return len(h.Events) > 0 } + +func (h *MatchRecorder) Handle(id uint, from, to uint64, flags uint, context interface{}) error { + if len(h.Events) > 0 { + tail := &h.Events[len(h.Events)-1] + + if tail.ID == id && tail.From == from && tail.ScanFlag == ScanFlag(flags) && tail.To < to { + tail.To = to + + return h.Err + } + } + + h.Events = append(h.Events, MatchEvent{id, from, to, ScanFlag(flags)}) + + return h.Err +} diff --git a/internal/hs/runtime_test.go b/internal/hs/runtime_test.go new file mode 100644 index 0000000..b66c725 --- /dev/null +++ b/internal/hs/runtime_test.go @@ -0,0 +1,112 @@ +package hs_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/internal/hs" +) + +func TestBlockScan(t *testing.T) { + Convey("Given a block database", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + db, err := hs.Compile("test", 0, hs.BlockMode, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + s, err := hs.AllocScratch(db) + + So(s, ShouldNotBeNil) + So(err, ShouldBeNil) + + h := &hs.MatchRecorder{} + + Convey("Scan block with pattern", func() { + So(hs.Scan(db, []byte("abctestdef"), 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldResemble, []hs.MatchEvent{{0, 0, 7, 0}}) + }) + + Convey("Scan block without pattern", func() { + So(hs.Scan(db, []byte("abcdef"), 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldBeEmpty) + }) + + Convey("Scan block with multi pattern", func() { + So(hs.Scan(db, []byte("abctestdeftest"), 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldResemble, []hs.MatchEvent{{0, 0, 14, 0}}) + }) + + Convey("Scan block with multi pattern but terminated", func() { + h.Err = hs.ErrScanTerminated + + So(hs.Scan(db, []byte("abctestdeftest"), 0, s, h.Handle, nil), ShouldEqual, hs.ErrScanTerminated) + So(h.Events, ShouldResemble, []hs.MatchEvent{{0, 0, 7, 0}}) + }) + + Convey("Scan empty buffers", func() { + So(hs.Scan(db, nil, 0, s, h.Handle, nil), ShouldEqual, hs.ErrInvalid) + So(hs.Scan(db, []byte(""), 0, s, h.Handle, nil), ShouldBeNil) + }) + + So(hs.FreeScratch(s), ShouldBeNil) + So(hs.FreeDatabase(db), ShouldBeNil) + }) +} + +func TestVectorScan(t *testing.T) { + Convey("Given a block database", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + db, err := hs.Compile("test", 0, hs.VectoredMode, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + s, err := hs.AllocScratch(db) + + So(s, ShouldNotBeNil) + So(err, ShouldBeNil) + + h := &hs.MatchRecorder{} + + Convey("Scan multi block with pattern", func() { + So(hs.ScanVector(db, [][]byte{[]byte("abctestdef"), []byte("abcdef")}, 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldResemble, []hs.MatchEvent{{0, 0, 7, 0}}) + }) + + Convey("Scan multi block without pattern", func() { + So(hs.ScanVector(db, [][]byte{[]byte("123456"), []byte("abcdef")}, 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldBeEmpty) + }) + + Convey("Scan multi block with multi pattern", func() { + So(hs.ScanVector(db, [][]byte{[]byte("abctestdef"), []byte("123test456")}, 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldResemble, []hs.MatchEvent{{0, 0, 17, 0}}) + }) + + Convey("Scan multi block with multi pattern but terminated", func() { + h.Err = hs.ErrScanTerminated + + So(hs.ScanVector(db, [][]byte{[]byte("abctestdef"), []byte("123test456")}, 0, s, h.Handle, nil), + ShouldEqual, hs.ErrScanTerminated) + So(h.Events, ShouldResemble, []hs.MatchEvent{{0, 0, 7, 0}}) + }) + + Convey("Scan empty buffers", func() { + So(hs.ScanVector(db, nil, 0, s, h.Handle, nil), ShouldEqual, hs.ErrInvalid) + So(hs.ScanVector(db, [][]byte{}, 0, s, h.Handle, nil), ShouldBeNil) + So(hs.ScanVector(db, [][]byte{[]byte(""), []byte("")}, 0, s, h.Handle, nil), ShouldBeNil) + }) + + So(hs.FreeScratch(s), ShouldBeNil) + }) +} diff --git a/internal/hs/scratch.go b/internal/hs/scratch.go new file mode 100644 index 0000000..739581c --- /dev/null +++ b/internal/hs/scratch.go @@ -0,0 +1,52 @@ +package hs + +// #include +import "C" + +type Scratch *C.hs_scratch_t + +func AllocScratch(db Database) (Scratch, error) { + var scratch *C.hs_scratch_t + + if ret := C.hs_alloc_scratch(db, &scratch); ret != C.HS_SUCCESS { + return nil, Error(ret) + } + + return scratch, nil +} + +func ReallocScratch(db Database, scratch *Scratch) error { + if ret := C.hs_alloc_scratch(db, (**C.struct_hs_scratch)(scratch)); ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func CloneScratch(scratch Scratch) (Scratch, error) { + var clone *C.hs_scratch_t + + if ret := C.hs_clone_scratch(scratch, &clone); ret != C.HS_SUCCESS { + return nil, Error(ret) + } + + return clone, nil +} + +func ScratchSize(scratch Scratch) (int, error) { + var size C.size_t + + if ret := C.hs_scratch_size(scratch, &size); ret != C.HS_SUCCESS { + return 0, Error(ret) + } + + return int(size), nil +} + +func FreeScratch(scratch Scratch) error { + if ret := C.hs_free_scratch(scratch); ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} diff --git a/internal/hs/scratch_test.go b/internal/hs/scratch_test.go new file mode 100644 index 0000000..ec47b95 --- /dev/null +++ b/internal/hs/scratch_test.go @@ -0,0 +1,76 @@ +package hs_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/hyperscan" + "github.com/flier/gohs/internal/hs" +) + +//nolint:funlen +func TestScratch(t *testing.T) { + Convey("Given a block database", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + db, err := hs.Compile("test", 0, hs.BlockMode, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Allocate a scratch", func() { + s, err := hs.AllocScratch(db) + + So(s, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the scratch size", func() { + size, err := hs.ScratchSize(s) + + So(size, ShouldBeGreaterThan, 1024) + So(size, ShouldBeLessThan, 4096) + So(err, ShouldBeNil) + + Convey("Clone the scratch", func() { + s2, err := hs.CloneScratch(s) + + So(s2, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Cloned scrash should have same size", func() { + size2, err := hs.ScratchSize(s2) + + So(size2, ShouldEqual, size) + So(err, ShouldBeNil) + }) + + So(hs.FreeScratch(s2), ShouldBeNil) + }) + + Convey("Reallocate the scratch with another database", func() { + db2, err := hs.Compile(hyperscan.EmailAddress, 0, hyperscan.BlockMode, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + So(hs.ReallocScratch(db2, &s), ShouldBeNil) + + size2, err := hs.ScratchSize(s) + + So(size2, ShouldBeGreaterThan, size) + So(err, ShouldBeNil) + + So(hs.FreeDatabase(db2), ShouldBeNil) + }) + }) + + So(hs.FreeScratch(s), ShouldBeNil) + }) + + So(hs.FreeDatabase(db), ShouldBeNil) + }) +} diff --git a/internal/hs/stream.go b/internal/hs/stream.go new file mode 100644 index 0000000..ddfe156 --- /dev/null +++ b/internal/hs/stream.go @@ -0,0 +1,174 @@ +package hs + +import ( + "reflect" + "runtime" + "unsafe" + + "github.com/flier/gohs/internal/handle" +) + +/* +#include + +extern int hsMatchEventCallback(unsigned int id, + unsigned long long from, + unsigned long long to, + unsigned int flags, + void *context); +*/ +import "C" + +type Stream *C.hs_stream_t + +func OpenStream(db Database, flags ScanFlag) (Stream, error) { + var stream *C.hs_stream_t + + if ret := C.hs_open_stream(db, C.uint(flags), &stream); ret != C.HS_SUCCESS { + return nil, Error(ret) + } + + return stream, nil +} + +func ScanStream(stream Stream, data []byte, flags ScanFlag, scratch Scratch, onEvent MatchEventHandler, context interface{}) error { + if data == nil { + return Error(C.HS_INVALID) + } + + h := handle.New(MatchEventContext{onEvent, context}) + defer h.Delete() + + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&data)) // FIXME: Zero-copy access to go data + + ret := C.hs_scan_stream(stream, + (*C.char)(unsafe.Pointer(hdr.Data)), + C.uint(hdr.Len), + C.uint(flags), + scratch, + C.match_event_handler(C.hsMatchEventCallback), + unsafe.Pointer(h)) + + // Ensure go data is alive before the C function returns + runtime.KeepAlive(data) + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func FreeStream(stream Stream) { + C.hs_close_stream(stream, nil, nil, nil) +} + +func CloseStream(stream Stream, scratch Scratch, onEvent MatchEventHandler, context interface{}) error { + h := handle.New(MatchEventContext{onEvent, context}) + defer h.Delete() + + ret := C.hs_close_stream(stream, + scratch, + C.match_event_handler(C.hsMatchEventCallback), + unsafe.Pointer(h)) + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func ResetStream(stream Stream, flags ScanFlag, scratch Scratch, onEvent MatchEventHandler, context interface{}) error { + h := handle.New(MatchEventContext{onEvent, context}) + defer h.Delete() + + ret := C.hs_reset_stream(stream, + C.uint(flags), + scratch, + C.match_event_handler(C.hsMatchEventCallback), + unsafe.Pointer(h)) + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func CopyStream(stream Stream) (Stream, error) { + var copied *C.hs_stream_t + + if ret := C.hs_copy_stream(&copied, stream); ret != C.HS_SUCCESS { + return nil, Error(ret) + } + + return copied, nil +} + +func ResetAndCopyStream(to, from Stream, scratch Scratch, onEvent MatchEventHandler, context interface{}) error { + h := handle.New(MatchEventContext{onEvent, context}) + defer h.Delete() + + ret := C.hs_reset_and_copy_stream(to, + from, + scratch, + C.match_event_handler(C.hsMatchEventCallback), + unsafe.Pointer(h)) + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func CompressStream(stream Stream, buf []byte) ([]byte, error) { + var size C.size_t + + ret := C.hs_compress_stream(stream, (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)), &size) + + if ret == C.HS_INSUFFICIENT_SPACE { + buf = make([]byte, size) + + ret = C.hs_compress_stream(stream, (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)), &size) + } + + if ret != C.HS_SUCCESS { + return nil, Error(ret) + } + + return buf[:size], nil +} + +func ExpandStream(db Database, stream *Stream, buf []byte) error { + ret := C.hs_expand_stream(db, (**C.hs_stream_t)(stream), (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf))) + + runtime.KeepAlive(buf) + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} + +func ResetAndExpandStream(stream Stream, buf []byte, scratch Scratch, onEvent MatchEventHandler, context interface{}) error { + h := handle.New(MatchEventContext{onEvent, context}) + defer h.Delete() + + ret := C.hs_reset_and_expand_stream(stream, + (*C.char)(unsafe.Pointer(&buf[0])), + C.size_t(len(buf)), + scratch, + C.match_event_handler(C.hsMatchEventCallback), + unsafe.Pointer(h)) + + runtime.KeepAlive(buf) + + if ret != C.HS_SUCCESS { + return Error(ret) + } + + return nil +} diff --git a/internal/hs/stream_test.go b/internal/hs/stream_test.go new file mode 100644 index 0000000..608c8e5 --- /dev/null +++ b/internal/hs/stream_test.go @@ -0,0 +1,96 @@ +package hs_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/internal/hs" +) + +//nolint:funlen +func TestStreamScan(t *testing.T) { + Convey("Given a stream database", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + db, err := hs.Compile("test", 0, hs.StreamMode, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + s, err := hs.AllocScratch(db) + + So(s, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Then open a stream", func() { + stream, err := hs.OpenStream(db, 0) + + So(stream, ShouldNotBeNil) + So(err, ShouldBeNil) + + h := &hs.MatchRecorder{} + + Convey("Then scan a simple stream with first part", func() { + So(hs.ScanStream(stream, []byte("abcte"), 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldBeNil) + + Convey("When scan second part, should be matched", func() { + So(hs.ScanStream(stream, []byte("stdef"), 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldResemble, []hs.MatchEvent{{0, 0, 7, 0}}) + }) + + Convey("Then copy the stream", func() { + stream2, err := hs.CopyStream(stream) + + So(stream2, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("When copied stream2 scan the second part, should be matched", func() { + So(hs.ScanStream(stream2, []byte("stdef"), 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldResemble, []hs.MatchEvent{{0, 0, 7, 0}}) + + Convey("When copied stream2 scan the second part again, should not be matched", func() { + h.Events = nil + So(hs.ScanStream(stream2, []byte("stdef"), 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldBeNil) + + Convey("When copy and reset stream2", func() { + So(hs.ResetAndCopyStream(stream2, stream, s, h.Handle, nil), ShouldBeNil) + + Convey("When copied and reset stream2 scan the second part again, should be matched", func() { + h.Events = nil + So(hs.ScanStream(stream2, []byte("stdef"), 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldResemble, []hs.MatchEvent{{0, 0, 7, 0}}) + }) + }) + }) + }) + + So(hs.CloseStream(stream2, s, h.Handle, nil), ShouldBeNil) + }) + + Convey("Then reset the stream", func() { + So(hs.ResetStream(stream, 0, s, h.Handle, nil), ShouldBeNil) + + Convey("When scan the second part, should not be matched", func() { + So(hs.ScanStream(stream, []byte("stdef"), 0, s, h.Handle, nil), ShouldBeNil) + So(h.Events, ShouldBeNil) + }) + + Convey("When scan empty buffers", func() { + So(hs.ScanStream(stream, nil, 0, s, h.Handle, nil), ShouldEqual, hs.ErrInvalid) + So(hs.ScanStream(stream, []byte(""), 0, s, h.Handle, nil), ShouldBeNil) + }) + }) + }) + + So(hs.CloseStream(stream, s, h.Handle, nil), ShouldBeNil) + }) + + So(hs.FreeScratch(s), ShouldBeNil) + }) +} From 8213a656ea40182fd46aaa8b1abd00e66be9bc7d Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sat, 25 Sep 2021 00:50:51 +0800 Subject: [PATCH 04/34] test chiema API --- .github/workflows/ci.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cab9eb1..7d23d1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,13 +44,17 @@ jobs: restore-keys: | ${{ runner.os }}-go- - - name: Test v4 API + - name: Test Hyperscan v4 API if: matrix.os == 'ubuntu-18.04' - run: go test -v -tags hyperscan_v4 ./... + run: go test -v -tags hyperscan_v4 ./internal/hs/... ./hyperscan/... - - name: Test v5 API + - name: Test Hyperscan v5 API if: matrix.os == 'ubuntu-20.04' || matrix.os == 'macos-latest' - run: go test -v ./... + run: go test -v ./internal/hs/... ./hyperscan/... + + - name: Test Chimera API + run: go test -v ./internal/ch/... ./chimera/... + continue-on-error: true golangci: name: lint From 310fe6380735517f5dd3f51f5c0d5ce4de368ff8 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sat, 25 Sep 2021 16:25:30 +0800 Subject: [PATCH 05/34] add section to build chimera --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 1c7ad56..a9dedbd 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,22 @@ Golang binding for Intel's HyperScan regex matching library: [hyperscan.io](http go get -u -tags hyperscan_v4 github.com/flier/gohs/hyperscan ``` +## Chimera [![Go Reference](https://pkg.go.dev/badge/github.com/flier/gohs/chimera.svg)](https://pkg.go.dev/github.com/flier/gohs/chimera) + +Chimera is a software regular expression matching engine that is a hybrid of Hyperscan and PCRE. The design goals of Chimera are to fully support PCRE syntax as well as to take advantage of the high performance nature of Hyperscan. + +It is recommended to compile and link Chimera using static libraries. + +```bash +$ mkdir build && cd build +$ cmake .. -G Ninja -DBUILD_STATIC_LIBS=on +$ ninja && ninja install +``` + +### Note + +You need to download the PCRE library source code to build Chimera, see [Chimera Requirements](https://intel.github.io/hyperscan/dev-reference/chimera.html#requirements) for more details + ## License This project is licensed under either of Apache License ([LICENSE-APACHE](LICENSE-APACHE)) or MIT license ([LICENSE-MIT](LICENSE-MIT)) at your option. From 5a1f40c77e4fa7c9b9d65de74cff59e2f2c340de Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sun, 26 Sep 2021 15:27:17 +0800 Subject: [PATCH 06/34] build with docker --- .dockerignore | 3 ++ Dockerfile | 61 ++++++++++++++++++++++++++++++++++++++++ chimera/common.go | 2 -- chimera/common_v54.go | 7 +++++ docker-compose.yml | 28 ++++++++++++++++++ internal/ch/error.go | 3 -- internal/ch/error_v54.go | 14 +++++++++ internal/ch/link.go | 4 +-- internal/hs/link.go | 2 +- 9 files changed, 116 insertions(+), 8 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 chimera/common_v54.go create mode 100644 docker-compose.yml create mode 100644 internal/ch/error_v54.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..28a4a13 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.github +examples +*.tar.gz diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f5c3b2a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,61 @@ +ARG UBUNTU_VERSION=20.04 + +FROM ubuntu:${UBUNTU_VERSION} + +ARG GO_VERSION=1.17.1 +ARG HYPERSCAN_VERSION=5.4.0 +ARG PCRE_VERSION=8.45 + +# Install dependencies + +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + cmake \ + libboost-dev \ + libbz2-dev \ + libpcap-dev \ + ninja-build \ + pkg-config \ + python2.7 \ + ragel \ + wget \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install golang + +RUN wget https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz -O /go${GO_VERSION}.tar.gz && \ + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf /go${GO_VERSION}.tar.gz && \ + rm /go${GO_VERSION}.tar.gz + +ENV PATH=/usr/local/go/bin:${PATH} + +# Download Hyperscan + +ENV HYPERSCAN_DIR=/hyperscan + +RUN wget https://github.com/intel/hyperscan/archive/refs/tags/v${HYPERSCAN_VERSION}.tar.gz -O /hyperscan-${HYPERSCAN_VERSION}.tar.gz && \ + mkdir ${HYPERSCAN_DIR} && tar xf /hyperscan-${HYPERSCAN_VERSION}.tar.gz -C ${HYPERSCAN_DIR} --strip-components=1 && rm /hyperscan-${HYPERSCAN_VERSION}.tar.gz +RUN wget https://sourceforge.net/projects/pcre/files/pcre/${PCRE_VERSION}/pcre-${PCRE_VERSION}.tar.gz/download -O /pcre-${PCRE_VERSION}.tar.gz && \ + mkdir ${HYPERSCAN_DIR}/pcre && tar xf /pcre-${PCRE_VERSION}.tar.gz -C ${HYPERSCAN_DIR}/pcre --strip-components=1 && rm /pcre-${PCRE_VERSION}.tar.gz + +# Install Hyperscan + +ENV INSTALL_DIR=/usr/local + +RUN mkdir ${HYPERSCAN_DIR}/build && cd ${HYPERSCAN_DIR}/build && \ + cmake -G Ninja -DBUILD_STATIC_LIBS=on -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} ${HYPERSCAN_DIR} && \ + ninja && ninja install && mv ${HYPERSCAN_DIR}/build/lib/lib*.a ${INSTALL_DIR}/lib/ && cd / && rm -rf ${HYPERSCAN_DIR} + +ENV PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig + +# Add gohs code + +ADD . /gohs/ + +WORKDIR /gohs +ENTRYPOINT ["/usr/local/go/bin/go"] +CMD ["test", "./..."] diff --git a/chimera/common.go b/chimera/common.go index 7ee1126..4cdc823 100644 --- a/chimera/common.go +++ b/chimera/common.go @@ -31,8 +31,6 @@ const ( ErrBadAlloc Error = ch.ErrBadAlloc // ErrScratchInUse is the error returned if the scratch region was already in use. ErrScratchInUse Error = ch.ErrScratchInUse - // ErrUnknown is the unexpected internal error from Hyperscan. - ErrUnknownHSError Error = ch.ErrUnknownHSError ) // DbInfo identify the version and platform information for the supplied database. diff --git a/chimera/common_v54.go b/chimera/common_v54.go new file mode 100644 index 0000000..c8cc95d --- /dev/null +++ b/chimera/common_v54.go @@ -0,0 +1,7 @@ +//go:build hyperscan_v54 +// +build hyperscan_v54 + +package chimera + +// ErrUnknown is the unexpected internal error from Hyperscan. +const ErrUnknownHSError Error = ch.ErrUnknownHSError diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1d2e015 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +version: "3.9" +services: + hyperscan-5.4: + build: + context: . + labels: + hyperscan: 5.4 + args: + HYPERSCAN_VERSION: 5.4.0 + command: test -tags hyperscan_v54 ./... + hyperscan-5.2: + build: + context: . + labels: + hyperscan: 5.2 + args: + HYPERSCAN_VERSION: 5.2.1 + hyperscan-4.7: + build: + context: . + labels: + hyperscan: 4.7 + args: + UBUNTU_VERSION: 18.04 + HYPERSCAN_VERSION: 4.7.0 + PCRE_VERSION: 8.41 + GO_BUILD_TAGS: -tags hyperscan_v4 + command: test -tags hyperscan_v4 ./hyperscan/... ./internal/hs/... diff --git a/internal/ch/error.go b/internal/ch/error.go index d127f1c..52d6b28 100644 --- a/internal/ch/error.go +++ b/internal/ch/error.go @@ -31,8 +31,6 @@ const ( ErrBadAlloc Error = C.CH_BAD_ALLOC // ErrScratchInUse is the error returned if the scratch region was already in use. ErrScratchInUse Error = C.CH_SCRATCH_IN_USE - // ErrUnknown is the unexpected internal error from Hyperscan. - ErrUnknownHSError Error = C.CH_UNKNOWN_HS_ERROR ) var ErrorMessages = map[Error]string{ @@ -47,7 +45,6 @@ var ErrorMessages = map[Error]string{ C.CH_BAD_ALIGN: "A parameter passed to this function was not correctly aligned.", C.CH_BAD_ALLOC: "The memory allocator did not correctly return aligned memory.", C.CH_SCRATCH_IN_USE: "The scratch region was already in use.", - C.CH_UNKNOWN_HS_ERROR: "Unexpected internal error from Hyperscan.", } func (e Error) Error() string { diff --git a/internal/ch/error_v54.go b/internal/ch/error_v54.go new file mode 100644 index 0000000..17e9086 --- /dev/null +++ b/internal/ch/error_v54.go @@ -0,0 +1,14 @@ +//go:build hyperscan_v54 +// +build hyperscan_v54 + +package ch + +// #include +import "C" + +// ErrUnknown is the unexpected internal error from Hyperscan. +const ErrUnknownHSError Error = C.CH_UNKNOWN_HS_ERROR + +func init() { + ErrorMessages[C.CH_UNKNOWN_HS_ERROR] = "Unexpected internal error from Hyperscan." +} diff --git a/internal/ch/link.go b/internal/ch/link.go index 9d3e6ee..13bce5e 100644 --- a/internal/ch/link.go +++ b/internal/ch/link.go @@ -1,8 +1,8 @@ package ch /* -#cgo pkg-config: libch libhs -#cgo linux LDFLAGS: -lm -lpcre +#cgo pkg-config: --static libch +#cgo linux LDFLAGS: -lm -lstdc++ -lpcre #cgo darwin LDFLAGS: -lpcre */ import "C" diff --git a/internal/hs/link.go b/internal/hs/link.go index b40e672..2632904 100644 --- a/internal/hs/link.go +++ b/internal/hs/link.go @@ -2,6 +2,6 @@ package hs /* #cgo pkg-config: libhs -#cgo linux LDFLAGS: -lm +#cgo linux LDFLAGS: -lm -lstdc++ */ import "C" From 48ed8bd387fa948a4d180903abd9f57f1aac643d Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sun, 26 Sep 2021 16:28:17 +0800 Subject: [PATCH 07/34] install hyperscan with flier/install-hyperscan@main --- .github/workflows/ci.yml | 143 ++++++++++++++++++++++++++++++++++---- chimera/common_v54.go | 2 + hyperscan/api.go | 2 +- hyperscan/platform_v54.go | 5 +- 4 files changed, 132 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d23d1c..07b1e04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,14 @@ name: Continuous integration -on: [push, pull_request] +on: + push: + branches: + - master + paths-ignore: + - "**.md" + pull_request: + paths-ignore: + - "**.md" defaults: run: @@ -11,35 +19,35 @@ jobs: strategy: matrix: os: [ubuntu-20.04, ubuntu-18.04, macos-latest] - go: [1.17.x, 1.16.x, 1.15.x, 1.14.x, 1.13.x] - name: Go ${{ matrix.go }} tests @ ${{ matrix.os }} + go: [1.17.x, 1.16.x, 1.15.x, 1.14.x] + name: Go ${{ matrix.go }} tests @ ${{ matrix.os }} for hyperscan ${{ matrix.hyperscan }} runs-on: ${{ matrix.os }} steps: - - name: Install Linux dependencies + - name: Install Linux dependencies for testing libraries if: startsWith(matrix.os, 'ubuntu-') run: | sudo apt-get update sudo apt-get install -yq libhyperscan-dev libpcap-dev - - name: Install MacOS dependencies + - name: Install MacOS dependencies for testing libraries if: startsWith(matrix.os, 'macos-') run: | - brew install pkg-config hyperscan libpcap + brew install hyperscan pkg-config libpcap + + - uses: actions/checkout@v2 - name: Install Golang ${{ matrix.go }} - id: install-golang uses: actions/setup-go@v2 with: go-version: ${{ matrix.go }} - - run: go version - - - name: Checkout code - uses: actions/checkout@v2 - name: Cache Golang modules uses: actions/cache@v2 with: - path: ~/go/pkg/mod + path: | + ~/.cache/go-build + ~/Library/Caches/go-build + ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- @@ -49,12 +57,115 @@ jobs: run: go test -v -tags hyperscan_v4 ./internal/hs/... ./hyperscan/... - name: Test Hyperscan v5 API - if: matrix.os == 'ubuntu-20.04' || matrix.os == 'macos-latest' + if: matrix.os != 'ubuntu-18.04' run: go test -v ./internal/hs/... ./hyperscan/... + build-and-test: + strategy: + matrix: + include: + - os: macos-latest + hyperscan: 5.4.0 + pcre: 8.45 + build_tags: -tags hyperscan_v54 + - os: macos-latest + hyperscan: 5.2.1 + pcre: 8.45 + - os: ubuntu-20.04 + hyperscan: 5.4.0 + pcre: 8.45 + build_tags: -tags hyperscan_v54 + - os: ubuntu-20.04 + hyperscan: 5.2.1 + pcre: 8.45 + - os: ubuntu-18.04 + hyperscan: 4.7.0 + pcre: 8.41 + build_tags: -tags hyperscan_v4 + name: Go ${{ matrix.go }} tests @ ${{ matrix.os }} for hyperscan ${{ matrix.hyperscan }} + runs-on: ${{ matrix.os }} + steps: + - name: Install Linux dependencies + if: startsWith(matrix.os, 'ubuntu-') + run: | + sudo apt-get update + sudo apt-get install -yq build-essential ca-certificates cmake libboost-dev libbz2-dev libpcap-dev \ + ninja-build pkg-config python2.7 ragel wget zlib1g-dev tree + + - name: Install MacOS dependencies + if: startsWith(matrix.os, 'macos-') + run: | + brew update + brew install pkg-config libpcap ragel cmake boost ninja lzlib wget tree + + - uses: actions/checkout@v2 + + - name: Cache Hyperscan library + id: cache-hyperscan + uses: actions/cache@v2 + env: + cache-name: cache-hyperscan-library + with: + path: ${{ github.workspace }}/dist + key: ${{ runner.os }}-build-hyperscan-${{ matrix.hyperscan }}-pcre-${{ matrix.pcre }} + restore-keys: | + ${{ runner.os }}-build-hyperscan-${{ matrix.hyperscan }}- + + - name: Install Hyperscan ${{ matrix.hyperscan }} with PCRE ${{ matrix.pcre }} + if: steps.cache-hyperscan.outputs.cache-hit == false + uses: flier/install-hyperscan@main + with: + hyperscan_version: ${{ matrix.hyperscan }} + pcre_version: ${{ matrix.pcre }} + build_static_lib: on + src_dir: ${{ runner.temp }}/hyperscan/ + install_prefix: ${{ github.workspace }}/dist/ + + - name: Upload Hyperscan ${{ matrix.hyperscan }} + if: steps.cache-hyperscan.outputs.cache-hit == false + uses: actions/upload-artifact@v2 + with: + name: hyperscan-${{ matrix.hyperscan }}-pcre-${{ matrix.pcre }} + path: ${{ github.workspace }}/dist/ + if-no-files-found: error + + - name: Check cached Hyperscan ${{ matrix.hyperscan}} with PCRE ${{ matrix.pcre }} + if: steps.cache-hyperscan.outputs.cache-hit + run: | + tree ${{ github.workspace }}/dist + + - name: Install Golang ${{ matrix.go }} + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Cache Golang modules + uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/Library/Caches/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Test Hyperscan API + env: + PKG_CONFIG_PATH: ${{ github.workspace }}/dist/lib/pkgconfig + CGO_CFLAGS: -I${{ github.workspace }}/dist/include/hs + CGO_LDFLAGS: -L${{ github.workspace }}/dist/lib + run: | + go test -v ${{ matrix.build_tags }} ./internal/hs/... ./hyperscan/... + - name: Test Chimera API - run: go test -v ./internal/ch/... ./chimera/... - continue-on-error: true + if: startsWith(matrix.hyperscan, '5.') + env: + PKG_CONFIG_PATH: ${{ github.workspace }}/dist/lib/pkgconfig + CGO_CFLAGS: -I${{ github.workspace }}/dist/include/hs + CGO_LDFLAGS: -L${{ github.workspace }}/dist/lib + run: | + go test -v ${{ matrix.build_tags }} ./internal/ch/... ./chimera/... golangci: name: lint @@ -64,7 +175,9 @@ jobs: run: | sudo apt-get update sudo apt-get install -yq libhyperscan-dev libpcap-dev + - uses: actions/checkout@v2 + - name: golangci-lint uses: golangci/golangci-lint-action@v2 with: diff --git a/chimera/common_v54.go b/chimera/common_v54.go index c8cc95d..7969160 100644 --- a/chimera/common_v54.go +++ b/chimera/common_v54.go @@ -3,5 +3,7 @@ package chimera +import "github.com/flier/gohs/internal/ch" + // ErrUnknown is the unexpected internal error from Hyperscan. const ErrUnknownHSError Error = ch.ErrUnknownHSError diff --git a/hyperscan/api.go b/hyperscan/api.go index f478469..c530499 100644 --- a/hyperscan/api.go +++ b/hyperscan/api.go @@ -74,6 +74,6 @@ func MatchReader(pattern string, reader io.Reader) (bool, error) { } // MatchString reports whether the string s contains any match of the regular expression pattern. -func MatchString(pattern string, s string) (matched bool, err error) { +func MatchString(pattern, s string) (matched bool, err error) { return Match(pattern, []byte(s)) } diff --git a/hyperscan/platform_v54.go b/hyperscan/platform_v54.go index 6f885b3..9a0c363 100644 --- a/hyperscan/platform_v54.go +++ b/hyperscan/platform_v54.go @@ -3,10 +3,7 @@ package hyperscan -/* -#include -*/ -import "C" +import "github.com/flier/gohs/internal/hs" const ( // AVX512VBMI is a CPU features flag indicates that the target platform From c23bd7f0616a32c73002b0c868ee623fbfa7b48c Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Mon, 27 Sep 2021 11:28:10 +0800 Subject: [PATCH 08/34] compile patterns with database builder --- chimera/api.go | 45 +++++++ chimera/api_test.go | 27 +++++ chimera/block.go | 208 ++++++++++++++++++++++++++++++++ chimera/common.go | 63 +++++++++- chimera/common_test.go | 72 +++++++++++ chimera/compile.go | 105 +++++++++++++++- chimera/compile_test.go | 95 +++++++++++++++ chimera/example_api_test.go | 20 +++ chimera/example_compile_test.go | 64 ++++++++++ chimera/example_runtime_test.go | 64 ++++++++++ chimera/pattern.go | 22 +++- chimera/runtime.go | 54 ++++++++- chimera/runtime_test.go | 91 ++++++++++++++ chimera/scratch.go | 60 +++++++++ hyperscan/block.go | 16 +++ hyperscan/common.go | 94 ++++----------- hyperscan/common_test.go | 24 ++-- hyperscan/scratch.go | 4 +- hyperscan/stream.go | 22 ++++ hyperscan/vectored.go | 16 +++ internal/ch/allocator.go | 24 +++- internal/ch/allocator_test.go | 106 ++++++++++++++++ internal/ch/common.go | 6 +- internal/ch/compile.go | 32 +++-- internal/ch/compile_test.go | 67 ++++++++++ internal/ch/runtime.go | 81 +++++++------ internal/ch/runtime_test.go | 65 ++++++++++ internal/ch/scratch.go | 52 ++++++++ internal/ch/scratch_test.go | 77 ++++++++++++ 29 files changed, 1525 insertions(+), 151 deletions(-) create mode 100644 chimera/api.go create mode 100644 chimera/api_test.go create mode 100644 chimera/block.go create mode 100644 chimera/compile_test.go create mode 100644 chimera/example_api_test.go create mode 100644 chimera/example_compile_test.go create mode 100644 chimera/example_runtime_test.go create mode 100644 chimera/runtime_test.go create mode 100644 chimera/scratch.go create mode 100644 internal/ch/allocator_test.go create mode 100644 internal/ch/compile_test.go create mode 100644 internal/ch/runtime_test.go create mode 100644 internal/ch/scratch.go create mode 100644 internal/ch/scratch_test.go diff --git a/chimera/api.go b/chimera/api.go new file mode 100644 index 0000000..3e730b7 --- /dev/null +++ b/chimera/api.go @@ -0,0 +1,45 @@ +package chimera + +import ( + "fmt" + + "github.com/flier/gohs/internal/ch" +) + +// Match reports whether the byte slice b contains any match of the regular expression pattern. +func Match(pattern string, data []byte) (bool, error) { + p, err := ParsePattern(pattern) + if err != nil { + return false, fmt.Errorf("parse pattern, %w", err) + } + + p.Flags |= SingleMatch + + db, err := NewBlockDatabase(p) + if err != nil { + return false, fmt.Errorf("create block database, %w", err) + } + defer db.Close() + + s, err := NewScratch(db) + if err != nil { + return false, fmt.Errorf("create scratch, %w", err) + } + + defer func() { + _ = s.Free() + }() + + h := &ch.MatchRecorder{} + + if err = db.Scan(data, s, h, nil); err != nil { + return false, err // nolint: wrapcheck + } + + return h.Matched(), h.Err +} + +// MatchString reports whether the string s contains any match of the regular expression pattern. +func MatchString(pattern, s string) (matched bool, err error) { + return Match(pattern, []byte(s)) +} diff --git a/chimera/api_test.go b/chimera/api_test.go new file mode 100644 index 0000000..6d5d770 --- /dev/null +++ b/chimera/api_test.go @@ -0,0 +1,27 @@ +package chimera_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/chimera" +) + +func TestMatch(t *testing.T) { + Convey("Given a compatible API for regexp package", t, func() { + Convey("When match a simple expression", func() { + matched, err := chimera.MatchString("test", "abctestdef") + + So(matched, ShouldBeTrue) + So(err, ShouldBeNil) + }) + + Convey("When match a invalid expression", func() { + matched, err := chimera.MatchString(`(?R`, "abctestdef") + + So(matched, ShouldBeFalse) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/chimera/block.go b/chimera/block.go new file mode 100644 index 0000000..e4ee01c --- /dev/null +++ b/chimera/block.go @@ -0,0 +1,208 @@ +package chimera + +import ( + "errors" + + "github.com/flier/gohs/internal/ch" +) + +// BlockScanner is the block (non-streaming) regular expression scanner. +type BlockScanner interface { + // This is the function call in which the actual pattern matching takes place for block-mode pattern databases. + Scan(data []byte, scratch *Scratch, handler Handler, context interface{}) error +} + +// BlockMatcher implements regular expression search. +type BlockMatcher interface { + // Find returns a slice holding the text of the leftmost match in b of the regular expression. + // A return value of nil indicates no match. + Find(data []byte) []byte + + // FindIndex returns a two-element slice of integers defining + // the location of the leftmost match in b of the regular expression. + // The match itself is at b[loc[0]:loc[1]]. A return value of nil indicates no match. + FindIndex(data []byte) []int + + // FindAll is the 'All' version of Find; it returns a slice of all successive matches of the expression, + // as defined by the 'All' description in the package comment. A return value of nil indicates no match. + FindAll(data []byte, n int) [][]byte + + // FindAllIndex is the 'All' version of FindIndex; it returns a slice of all successive matches of the expression, + // as defined by the 'All' description in the package comment. A return value of nil indicates no match. + FindAllIndex(data []byte, n int) [][]int + + // FindString returns a string holding the text of the leftmost match in s of the regular expression. + // If there is no match, the return value is an empty string, but it will also be empty + // if the regular expression successfully matches an empty string. + // Use FindStringIndex if it is necessary to distinguish these cases. + FindString(s string) string + + // FindStringIndex returns a two-element slice of integers defining + // the location of the leftmost match in s of the regular expression. + // The match itself is at s[loc[0]:loc[1]]. A return value of nil indicates no match. + FindStringIndex(s string) []int + + // FindAllString is the 'All' version of FindString; it returns a slice of all successive matches of the expression, + // as defined by the 'All' description in the package comment. A return value of nil indicates no match. + FindAllString(s string, n int) []string + + // FindAllStringIndex is the 'All' version of FindStringIndex; + // it returns a slice of all successive matches of the expression, + // as defined by the 'All' description in the package comment. A return value of nil indicates no match. + FindAllStringIndex(s string, n int) [][]int + + // Match reports whether the pattern database matches the byte slice b. + Match(b []byte) bool + + // MatchString reports whether the pattern database matches the string s. + MatchString(s string) bool +} + +// BlockDatabase scan the target data that is a discrete, +// contiguous block which can be scanned in one call and does not require state to be retained. +type BlockDatabase interface { + Database + BlockScanner + BlockMatcher +} + +type blockDatabase struct { + *blockMatcher +} + +func newBlockDatabase(db ch.Database) *blockDatabase { + return &blockDatabase{newBlockMatcher(newBlockScanner(newDatabase(db)))} +} + +type blockScanner struct { + *baseDatabase +} + +func newBlockScanner(bdb *baseDatabase) *blockScanner { + return &blockScanner{bdb} +} + +func (bs *blockScanner) Scan(data []byte, s *Scratch, h Handler, ctx interface{}) (err error) { + if s == nil { + s, err = NewScratch(bs) + + if err != nil { + return + } + + defer func() { + _ = s.Free() + }() + } + + return ch.Scan(bs.db, data, 0, s.s, h.OnMatch, h.OnError, ctx) // nolint: wrapcheck +} + +type blockMatcher struct { + *blockScanner + *ch.MatchRecorder + n int +} + +func newBlockMatcher(scanner *blockScanner) *blockMatcher { + return &blockMatcher{blockScanner: scanner} +} + +func (m *blockMatcher) OnMatch(id uint, from, to uint64, flags uint, + captured []*Capture, context interface{}) (r Callback) { + r = m.MatchRecorder.OnMatch(id, from, to, flags, captured, context) + + if m.n < 0 { + return Continue + } + + if m.n < len(m.Events) { + m.Events = m.Events[:m.n] + + return Terminate + } + + return +} + +func (m *blockMatcher) scan(data []byte) error { + m.MatchRecorder = &ch.MatchRecorder{} + + return m.blockScanner.Scan(data, nil, m, nil) +} + +const findIndexMatches = 2 + +func (m *blockMatcher) Find(data []byte) []byte { + if loc := m.FindIndex(data); len(loc) == findIndexMatches { + return data[loc[0]:loc[1]] + } + + return nil +} + +func (m *blockMatcher) FindIndex(data []byte) []int { + if m.Match(data) && len(m.Events) == 1 { + return []int{int(m.Events[0].From), int(m.Events[0].To)} + } + + return nil +} + +func (m *blockMatcher) FindAll(data []byte, n int) (matches [][]byte) { + if locs := m.FindAllIndex(data, n); len(locs) > 0 { + for _, loc := range locs { + matches = append(matches, data[loc[0]:loc[1]]) + } + } + + return +} + +func (m *blockMatcher) FindAllIndex(data []byte, n int) (locs [][]int) { + if n < 0 { + n = len(data) + 1 + } + + m.n = n + + if err := m.scan(data); (err == nil || errors.Is(err, ErrScanTerminated)) && len(m.Events) > 0 { + for _, e := range m.Events { + locs = append(locs, []int{int(e.From), int(e.To)}) + } + } + + return +} + +func (m *blockMatcher) FindString(s string) string { + return string(m.Find([]byte(s))) +} + +func (m *blockMatcher) FindStringIndex(s string) (loc []int) { + return m.FindIndex([]byte(s)) +} + +func (m *blockMatcher) FindAllString(s string, n int) (results []string) { + for _, m := range m.FindAll([]byte(s), n) { + results = append(results, string(m)) + } + + return +} + +func (m *blockMatcher) FindAllStringIndex(s string, n int) [][]int { + return m.FindAllIndex([]byte(s), n) +} + +func (m *blockMatcher) Match(data []byte) bool { + m.n = 1 + + err := m.scan(data) + + return (err == nil || errors.Is(err, ErrScanTerminated)) && len(m.Events) == m.n +} + +func (m *blockMatcher) MatchString(s string) bool { + return m.Match([]byte(s)) +} diff --git a/chimera/common.go b/chimera/common.go index 4cdc823..c96d493 100644 --- a/chimera/common.go +++ b/chimera/common.go @@ -1,6 +1,10 @@ package chimera import ( + "fmt" + "regexp" + + "github.com/flier/gohs/hyperscan" "github.com/flier/gohs/internal/ch" ) @@ -33,11 +37,54 @@ const ( ErrScratchInUse Error = ch.ErrScratchInUse ) +// ModeFlag represents the compile mode flags. +type ModeFlag = hyperscan.ModeFlag + +// BlockMode for the block scan (non-streaming) database. +const BlockMode = hyperscan.BlockMode + // DbInfo identify the version and platform information for the supplied database. type DbInfo string // nolint: stylecheck +// parse `Chimera Version: 5.4.0 Features: AVX2 Mode: BLOCK`. +var regexDBInfo = regexp.MustCompile(`^Chimera Version: (\d+\.\d+\.\d+) Features: ([\w\s]+)? Mode: (\w+)$`) + +const dbInfoMatches = 4 + func (i DbInfo) String() string { return string(i) } +// Parse the version and platform information. +func (i DbInfo) Parse() (version, features, mode string, err error) { + matched := regexDBInfo.FindStringSubmatch(string(i)) + + if len(matched) != dbInfoMatches { + err = fmt.Errorf("database info `%s`, %w", i, ErrInvalid) + } else { + version = matched[1] + features = matched[2] + mode = matched[3] + } + + return +} + +// Version is the version for the supplied database. +func (i DbInfo) Version() (string, error) { + version, _, _, err := i.Parse() + + return version, err +} + +// Mode is the scanning mode for the supplied database. +func (i DbInfo) Mode() (ModeFlag, error) { + _, _, mode, err := i.Parse() + if err != nil { + return 0, err + } + + return hyperscan.ParseModeFlag(mode) // nolint: wrapcheck +} + // Database is an immutable database that can be used by the Hyperscan scanning API. type Database interface { // Provides information about a database. @@ -50,13 +97,21 @@ type Database interface { Close() error } -type database struct { +type database interface { + c() ch.Database +} + +type baseDatabase struct { db ch.Database } -func (d *database) Size() (int, error) { return ch.DatabaseSize(d.db) } // nolint: wrapcheck +func newDatabase(db ch.Database) *baseDatabase { return &baseDatabase{db} } + +func (d *baseDatabase) c() ch.Database { return d.db } + +func (d *baseDatabase) Size() (int, error) { return ch.DatabaseSize(d.db) } // nolint: wrapcheck -func (d *database) Info() (DbInfo, error) { +func (d *baseDatabase) Info() (DbInfo, error) { i, err := ch.DatabaseInfo(d.db) if err != nil { return "", err //nolint: wrapcheck @@ -65,7 +120,7 @@ func (d *database) Info() (DbInfo, error) { return DbInfo(i), nil } -func (d *database) Close() error { return ch.FreeDatabase(d.db) } // nolint: wrapcheck +func (d *baseDatabase) Close() error { return ch.FreeDatabase(d.db) } // nolint: wrapcheck // Version identify this release version. // diff --git a/chimera/common_test.go b/chimera/common_test.go index 6e25829..99fd7c0 100644 --- a/chimera/common_test.go +++ b/chimera/common_test.go @@ -13,3 +13,75 @@ func TestChimera(t *testing.T) { So(chimera.Version(), ShouldNotBeEmpty) }) } + +func TestBaseDatabase(t *testing.T) { + Convey("Given a block database", t, func() { + So(chimera.ValidPlatform(), ShouldBeNil) + + bdb, err := chimera.NewBlockDatabase(&chimera.Pattern{Expression: "test"}) + + So(err, ShouldBeNil) + So(bdb, ShouldNotBeNil) + + Convey("When get size", func() { + size, err := bdb.Size() + + So(err, ShouldBeNil) + So(size, ShouldBeGreaterThan, 800) + }) + + Convey("When get info", func() { + info, err := bdb.Info() + + So(err, ShouldBeNil) + So(info, ShouldNotBeNil) + + _, _, _, err = info.Parse() + So(err, ShouldBeNil) + + Convey("Then get version", func() { + ver, err := info.Version() + + So(err, ShouldBeNil) + So(chimera.Version(), ShouldStartWith, ver) + }) + + Convey("Then get mode", func() { + mode, err := info.Mode() + + So(err, ShouldBeNil) + So(mode, ShouldEqual, chimera.BlockMode) + }) + }) + + So(bdb.Close(), ShouldBeNil) + }) +} + +func TestBlockDatabase(t *testing.T) { + Convey("Give a block database", t, func() { + bdb, err := chimera.NewBlockDatabase(&chimera.Pattern{Expression: "test"}) + + So(err, ShouldBeNil) + So(bdb, ShouldNotBeNil) + + Convey("When get info", func() { + info, err := bdb.Info() + + So(err, ShouldBeNil) + So(info, ShouldNotBeNil) + + _, _, _, err = info.Parse() + So(err, ShouldBeNil) + + Convey("Then get mode", func() { + mode, err := info.Mode() + + So(err, ShouldBeNil) + So(mode, ShouldEqual, chimera.BlockMode) + }) + }) + + So(bdb.Close(), ShouldBeNil) + }) +} diff --git a/chimera/compile.go b/chimera/compile.go index c044ffd..7a52170 100644 --- a/chimera/compile.go +++ b/chimera/compile.go @@ -2,9 +2,11 @@ package chimera import ( "fmt" + "runtime" "strconv" "github.com/flier/gohs/internal/ch" + "github.com/flier/gohs/internal/hs" ) // A type containing error details that is returned by the compile calls on failure. @@ -65,6 +67,107 @@ const ( Groups CompileMode = ch.Groups ) +type Builder interface { + Build(mode CompileMode) (Database, error) + + ForPlatform(mode CompileMode, platform Platform) (Database, error) +} + +func (p *Pattern) Build(mode CompileMode) (Database, error) { + return p.ForPlatform(mode, nil) +} + +func (p *Pattern) ForPlatform(mode CompileMode, platform Platform) (Database, error) { + b := DatabaseBuilder{Patterns: Patterns{p}, Mode: mode, Platform: platform} + return b.Build() +} + +func (p Patterns) Build(mode CompileMode) (Database, error) { + return p.ForPlatform(mode, nil) +} + +func (p Patterns) ForPlatform(mode CompileMode, platform Platform) (Database, error) { + b := DatabaseBuilder{Patterns: p, Mode: mode, Platform: platform} + return b.Build() +} + +// DatabaseBuilder to help to build up a database. +type DatabaseBuilder struct { + // Array of patterns to compile. + Patterns + + // Compiler mode flags that affect the database as a whole. (Default: capturing groups mode) + Mode CompileMode + + // If not nil, the platform structure is used to determine the target platform for the database. + // If nil, a database suitable for running on the current host platform is produced. + Platform + + // A limit from pcre_extra on the amount of match function called in PCRE to limit backtracking that can take place. + MatchLimit uint + + // A limit from pcre_extra on the recursion depth of match function in PCRE. + MatchLimitRecursion uint +} + +// AddExpressions add more expressions to the database. +func (b *DatabaseBuilder) AddExpressions(exprs ...Expression) *DatabaseBuilder { + for _, expr := range exprs { + b.Patterns = append(b.Patterns, &Pattern{Expression: expr, Id: len(b.Patterns) + 1}) + } + + return b +} + +// AddExpressionWithFlags add more expressions with flags to the database. +func (b *DatabaseBuilder) AddExpressionWithFlags(expr Expression, flags CompileFlag) *DatabaseBuilder { + b.Patterns = append(b.Patterns, &Pattern{Expression: expr, Flags: flags, Id: len(b.Patterns) + 1}) + + return b +} + +// Build a database base on the expressions and platform. +func (b *DatabaseBuilder) Build() (Database, error) { + if b.Patterns == nil { + return nil, ErrInvalid + } + + platform, _ := b.Platform.(*hs.PlatformInfo) + + db, err := ch.CompileExtMulti(b.Patterns, b.Mode, platform, b.MatchLimit, b.MatchLimitRecursion) + if err != nil { + return nil, err // nolint: wrapcheck + } + + return newBlockDatabase(db), nil +} + +// NewBlockDatabase compile expressions into a pattern database. +func NewBlockDatabase(patterns ...*Pattern) (BlockDatabase, error) { + db, err := Patterns(patterns).Build(Groups) + if err != nil { + return nil, err + } + + return db.(BlockDatabase), err +} + +// NewManagedBlockDatabase is a wrapper for NewBlockDatabase that +// sets a finalizer on the Scratch instance so that memory is +// freed once the object is no longer in use. +func NewManagedBlockDatabase(patterns ...*Pattern) (BlockDatabase, error) { + db, err := NewBlockDatabase(patterns...) + if err != nil { + return nil, err + } + + runtime.SetFinalizer(db, func(obj BlockDatabase) { + _ = obj.Close() + }) + + return db, nil +} + // Compile a regular expression and returns, if successful, // a pattern database in the block mode that can be used to match against text. func Compile(expr string) (Database, error) { @@ -73,7 +176,7 @@ func Compile(expr string) (Database, error) { return nil, err // nolint: wrapcheck } - return &database{db}, nil + return newBlockDatabase(db), nil } // MustCompile is like Compile but panics if the expression cannot be parsed. diff --git a/chimera/compile_test.go b/chimera/compile_test.go new file mode 100644 index 0000000..b896e81 --- /dev/null +++ b/chimera/compile_test.go @@ -0,0 +1,95 @@ +package chimera_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/chimera" + "github.com/flier/gohs/hyperscan" +) + +func TestCompileFlag(t *testing.T) { + Convey("Given a compile flags", t, func() { + flags := chimera.Caseless | chimera.DotAll | chimera.MultiLine | chimera.SingleMatch | + chimera.Utf8Mode | chimera.UnicodeProperty + + So(flags.String(), ShouldEqual, "8HWims") + + Convey("When parse valid flags", func() { + f, err := chimera.ParseCompileFlag("ismH8W") + + So(f, ShouldEqual, flags) + So(err, ShouldBeNil) + }) + + Convey("When parse invalid flags", func() { + f, err := chimera.ParseCompileFlag("abc") + + So(f, ShouldEqual, 0) + So(err, ShouldNotBeNil) + }) + }) +} + +func TestDatabaseBuilder(t *testing.T) { + Convey("Given a DatabaseBuilder", t, func() { + b := chimera.DatabaseBuilder{} + + Convey("When build without patterns", func() { + db, err := b.Build() + + So(db, ShouldBeNil) + So(err, ShouldNotBeNil) + }) + + Convey("When build with a simple expression", func() { + db, err := b.AddExpressions("test").Build() + + So(err, ShouldBeNil) + So(db, ShouldNotBeNil) + + So(db.Close(), ShouldBeNil) + }) + + Convey("When build with some complicated expression", func() { + db, err := b.AddExpressions("test", hyperscan.EmailAddress, hyperscan.IPv4Address, hyperscan.CreditCard).Build() + + So(err, ShouldBeNil) + So(db, ShouldNotBeNil) + + info, err := db.Info() + + So(err, ShouldBeNil) + + mode, err := info.Mode() + + So(err, ShouldBeNil) + So(mode, ShouldEqual, chimera.BlockMode) + + So(db.Close(), ShouldBeNil) + }) + }) +} + +func TestCompile(t *testing.T) { + Convey("Given compile some expressions", t, func() { + Convey("When compile a simple expression", func() { + db, err := chimera.Compile("test") + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + So(db.Close(), ShouldBeNil) + }) + + Convey("When compile a complex expression", func() { + db, err := chimera.Compile(hyperscan.CreditCard) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + So(db.Close(), ShouldBeNil) + }) + }) +} diff --git a/chimera/example_api_test.go b/chimera/example_api_test.go new file mode 100644 index 0000000..3102152 --- /dev/null +++ b/chimera/example_api_test.go @@ -0,0 +1,20 @@ +package chimera_test + +import ( + "fmt" + + "github.com/flier/gohs/chimera" +) + +func ExampleMatch() { + matched, err := chimera.Match(`foo.*`, []byte(`seafood`)) + fmt.Println(matched, err) + matched, err = chimera.Match(`bar.*`, []byte(`seafood`)) + fmt.Println(matched, err) + matched, err = chimera.Match(`a(b`, []byte(`seafood`)) + fmt.Println(matched, err) + // Output: + // true + // false + // false create block database, PCRE compilation failed: missing ). +} diff --git a/chimera/example_compile_test.go b/chimera/example_compile_test.go new file mode 100644 index 0000000..a0d039e --- /dev/null +++ b/chimera/example_compile_test.go @@ -0,0 +1,64 @@ +package chimera_test + +import ( + "fmt" + "strings" + + "github.com/flier/gohs/chimera" +) + +// This example demonstrates construct and match a pattern. +func ExamplePattern() { + p := chimera.NewPattern(`foo.*bar`, chimera.Caseless) + fmt.Println(p) + + db, err := chimera.NewBlockDatabase(p) + fmt.Println(err) + + found := db.MatchString("fooxyzbarbar") + fmt.Println(found) + + // Output: + // /foo.*bar/i + // + // true +} + +// This example demonstrates parsing pattern with id and flags. +func ExampleParsePattern() { + p, err := chimera.ParsePattern("3:/foobar/i8") + + fmt.Println(err) + fmt.Println(p.Id) + fmt.Println(p.Expression) + fmt.Println(p.Flags) + + // Output: + // + // 3 + // foobar + // 8i +} + +// This example demonstrates parsing patterns with comment. +func ExampleParsePatterns() { + patterns, err := chimera.ParsePatterns(strings.NewReader(` +# empty line and comment will be skipped + +1:/hatstand.*teakettle/s +2:/(hatstand|teakettle)/iH +3:/^.{10,20}hatstand/m +`)) + + fmt.Println(err) + + for _, p := range patterns { + fmt.Println(p) + } + + // Output: + // + // 1:/hatstand.*teakettle/s + // 2:/(hatstand|teakettle)/Hi + // 3:/^.{10,20}hatstand/m +} diff --git a/chimera/example_runtime_test.go b/chimera/example_runtime_test.go new file mode 100644 index 0000000..542d24b --- /dev/null +++ b/chimera/example_runtime_test.go @@ -0,0 +1,64 @@ +package chimera_test + +import ( + "fmt" + + "github.com/flier/gohs/chimera" +) + +func ExampleBlockScanner() { + p, err := chimera.ParsePattern(`foo(bar)+`) + if err != nil { + fmt.Println("parse pattern failed,", err) + return + } + + // Create new block database with pattern + db, err := chimera.NewBlockDatabase(p) + if err != nil { + fmt.Println("create database failed,", err) + return + } + defer db.Close() + + // Create new scratch for scanning + s, err := chimera.NewScratch(db) + if err != nil { + fmt.Println("create scratch failed,", err) + return + } + + defer func() { + _ = s.Free() + }() + + // Record matching text + type Match struct { + from uint64 + to uint64 + } + + var matches []Match + + handler := chimera.MatchHandlerFunc(func(id uint, from, to uint64, flags uint, + captured []*chimera.Capture, ctx interface{}) chimera.Callback { + matches = append(matches, Match{from, to}) + return chimera.Continue + }) + + data := []byte("hello foobarbar!") + + // Scan data block with handler + if err := db.Scan(data, s, handler, nil); err != nil { + fmt.Println("database scan failed,", err) + return + } + + // chimera will reports all matches + for _, m := range matches { + fmt.Println("match [", m.from, ":", m.to, "]", string(data[m.from:m.to])) + } + + // Output: + // match [ 6 : 15 ] foobarbar +} diff --git a/chimera/pattern.go b/chimera/pattern.go index 8e6c07a..0d4d916 100644 --- a/chimera/pattern.go +++ b/chimera/pattern.go @@ -14,7 +14,7 @@ import ( type Expression = ch.Expression // Pattern is a matching pattern. -type Pattern = ch.Pattern +type Pattern ch.Pattern // NewPattern returns a new pattern base on expression and compile flags. func NewPattern(expr string, flags CompileFlag) *Pattern { @@ -64,6 +64,18 @@ func ParsePattern(s string) (*Pattern, error) { return &p, nil } +func (p *Pattern) String() string { + var b strings.Builder + + if p.Id > 0 { + fmt.Fprintf(&b, "%d:", p.Id) + } + + fmt.Fprintf(&b, "/%s/%s", p.Expression, p.Flags) + + return b.String() +} + // Patterns is a set of matching patterns. type Patterns []*Pattern @@ -94,3 +106,11 @@ func ParsePatterns(r io.Reader) (patterns Patterns, err error) { return } + +func (p Patterns) Patterns() (r []*ch.Pattern) { + r = make([]*ch.Pattern, len(p)) + for i, pat := range p { + r[i] = (*ch.Pattern)(pat) + } + return +} diff --git a/chimera/runtime.go b/chimera/runtime.go index dd12e82..5165599 100644 --- a/chimera/runtime.go +++ b/chimera/runtime.go @@ -1,6 +1,54 @@ package chimera -import "github.com/flier/gohs/internal/ch" +import ( + "github.com/flier/gohs/hyperscan" + "github.com/flier/gohs/internal/ch" +) -// A Chimera scratch space. -type Scratch = ch.Scratch +// Platform is a type containing information on the target platform. +type Platform = hyperscan.Platform + +// ValidPlatform test the current system architecture. +var ValidPlatform = hyperscan.ValidPlatform + +// Callback return value used to tell the Chimera matcher what to do after processing this match. +type Callback = ch.Callback + +const ( + Continue Callback = ch.Continue // Continue matching. + Terminate Callback = ch.Terminate // Terminate matching. + SkipPattern Callback = ch.SkipPattern // Skip remaining matches for this ID and continue. +) + +// Capture representing a captured subexpression within a match. +type Capture = ch.Capture + +// Type used to differentiate the errors raised with the `ErrorEventHandler` callback. +type ErrorEvent = ch.ErrorEvent + +const ( + // PCRE hits its match limit and reports PCRE_ERROR_MATCHLIMIT. + ErrMatchLimit ErrorEvent = ch.ErrMatchLimit + // PCRE hits its recursion limit and reports PCRE_ERROR_RECURSIONLIMIT. + ErrRecursionLimit ErrorEvent = ch.ErrRecursionLimit +) + +// Definition of the chimera event callback handler. +type Handler interface { + // OnMatch will be invoked whenever a match is located in the target data during the execution of a scan. + OnMatch(id uint, from, to uint64, flags uint, captured []*Capture, context interface{}) Callback + + // OnError will be invoked when an error event occurs during matching; + // this indicates that some matches for a given expression may not be reported. + OnError(event ErrorEvent, id uint, info, context interface{}) Callback +} + +type MatchHandlerFunc func(id uint, from, to uint64, flags uint, captured []*Capture, context interface{}) Callback + +func (f MatchHandlerFunc) OnMatch(id uint, from, to uint64, flags uint, captured []*Capture, ctx interface{}) Callback { + return f(id, from, to, flags, captured, ctx) +} + +func (f MatchHandlerFunc) OnError(event ErrorEvent, id uint, info, context interface{}) Callback { + return Terminate +} diff --git a/chimera/runtime_test.go b/chimera/runtime_test.go new file mode 100644 index 0000000..2457db4 --- /dev/null +++ b/chimera/runtime_test.go @@ -0,0 +1,91 @@ +package chimera_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/chimera" +) + +type ( + BlockDatabaseConstructor func(patterns ...*chimera.Pattern) (chimera.BlockDatabase, error) +) + +var blockDatabaseConstructors = map[string]BlockDatabaseConstructor{ + "normal": chimera.NewBlockDatabase, + "managed": chimera.NewManagedBlockDatabase, +} + +func TestBlockScanner(t *testing.T) { + for dbType, dbConstructor := range blockDatabaseConstructors { + Convey("Given a "+dbType+" block database", t, func() { + bdb, err := dbConstructor(chimera.NewPattern(`\d+`, 0)) // nolint: scopelint + + So(err, ShouldBeNil) + So(bdb, ShouldNotBeNil) + + Convey("When scan a string", func() { + var matches [][]uint64 + + matched := func(id uint, from, to uint64, flags uint, + captured []*chimera.Capture, context interface{}) chimera.Callback { + matches = append(matches, []uint64{from, to}) + + return chimera.Continue + } + + err = bdb.Scan([]byte("abc123def456"), nil, chimera.MatchHandlerFunc(matched), nil) + + So(err, ShouldBeNil) + So(matches, ShouldResemble, [][]uint64{{3, 6}, {9, 12}}) + }) + }) + } +} + +func TestBlockMatcher(t *testing.T) { + for dbType, dbConstructor := range blockDatabaseConstructors { + Convey("Given a "+dbType+" block database", t, func() { + bdb, err := dbConstructor(chimera.NewPattern(`\d+`, 0)) // nolint: scopelint + + So(err, ShouldBeNil) + So(bdb, ShouldNotBeNil) + + Convey("When match the string", func() { + So(bdb.MatchString("123"), ShouldBeTrue) + So(bdb.MatchString("abc123def456"), ShouldBeTrue) + }) + + Convey("When find the leftmost matched string index", func() { + So(bdb.FindStringIndex("123"), ShouldResemble, []int{0, 3}) + So(bdb.FindStringIndex("abc123def456"), ShouldResemble, []int{3, 6}) + }) + + Convey("When find the leftmost matched string", func() { + So(bdb.FindString("123"), ShouldEqual, "123") + So(bdb.FindString("abc123def456"), ShouldEqual, "123") + }) + + Convey("When find all the matched string index", func() { + So(bdb.FindAllStringIndex("123", -1), ShouldResemble, [][]int{{0, 3}}) + So(bdb.FindAllStringIndex("abc123def456", -1), ShouldResemble, [][]int{{3, 6}, {9, 12}}) + }) + + Convey("When find all the matched string", func() { + So(bdb.FindAllString("abc123def456", -1), ShouldResemble, + []string{"123", "456"}) + }) + + Convey("When find all the first 4 matched string index", func() { + So(bdb.FindAllStringIndex("abc123def456", 1), ShouldResemble, + [][]int{{3, 6}}) + }) + + Convey("When find all the first 4 matched string", func() { + So(bdb.FindAllString("abc123def456", 1), ShouldResemble, + []string{"123"}) + }) + }) + } +} diff --git a/chimera/scratch.go b/chimera/scratch.go new file mode 100644 index 0000000..dbbfd0f --- /dev/null +++ b/chimera/scratch.go @@ -0,0 +1,60 @@ +package chimera + +import ( + "runtime" + + "github.com/flier/gohs/internal/ch" +) + +// Scratch is a Chimera scratch space. +type Scratch struct { + s ch.Scratch +} + +// NewScratch allocate a "scratch" space for use by Chimera. +// This is required for runtime use, and one scratch space per thread, +// or concurrent caller, is required. +func NewScratch(db Database) (*Scratch, error) { + s, err := ch.AllocScratch(db.(database).c()) + if err != nil { + return nil, err // nolint: wrapcheck + } + + return &Scratch{s}, nil +} + +// NewManagedScratch is a wrapper for NewScratch that sets +// a finalizer on the Scratch instance so that memory is freed +// once the object is no longer in use. +func NewManagedScratch(db Database) (*Scratch, error) { + s, err := NewScratch(db) + if err != nil { + return nil, err + } + + runtime.SetFinalizer(s, func(scratch *Scratch) { + _ = scratch.Free() + }) + return s, nil +} + +// Size provides the size of the given scratch space. +func (s *Scratch) Size() (int, error) { return ch.ScratchSize(s.s) } // nolint: wrapcheck + +// Realloc reallocate the scratch for another database. +func (s *Scratch) Realloc(db Database) error { + return ch.ReallocScratch(db.(database).c(), &s.s) // nolint: wrapcheck +} + +// Clone allocate a scratch space that is a clone of an existing scratch space. +func (s *Scratch) Clone() (*Scratch, error) { + cloned, err := ch.CloneScratch(s.s) + if err != nil { + return nil, err // nolint: wrapcheck + } + + return &Scratch{cloned}, nil +} + +// Free a scratch block previously allocated. +func (s *Scratch) Free() error { return ch.FreeScratch(s.s) } // nolint: wrapcheck diff --git a/hyperscan/block.go b/hyperscan/block.go index 651f19c..f7c1bd0 100644 --- a/hyperscan/block.go +++ b/hyperscan/block.go @@ -58,6 +58,22 @@ type BlockMatcher interface { MatchString(s string) bool } +// BlockDatabase scan the target data that is a discrete, +// contiguous block which can be scanned in one call and does not require state to be retained. +type BlockDatabase interface { + Database + BlockScanner + BlockMatcher +} + +type blockDatabase struct { + *blockMatcher +} + +func newBlockDatabase(db hs.Database) *blockDatabase { + return &blockDatabase{newBlockMatcher(newBlockScanner(newBaseDatabase(db)))} +} + type blockScanner struct { *baseDatabase } diff --git a/hyperscan/common.go b/hyperscan/common.go index 674b996..8f87914 100644 --- a/hyperscan/common.go +++ b/hyperscan/common.go @@ -56,63 +56,45 @@ type Database interface { Unmarshal([]byte) error } -// BlockDatabase scan the target data that is a discrete, -// contiguous block which can be scanned in one call and does not require state to be retained. -type BlockDatabase interface { - Database - BlockScanner - BlockMatcher -} - -// StreamDatabase scan the target data to be scanned is a continuous stream, -// not all of which is available at once; -// blocks of data are scanned in sequence and matches may span multiple blocks in a stream. -type StreamDatabase interface { - Database - StreamScanner - StreamMatcher - StreamCompressor +// DbInfo identify the version and platform information for the supplied database. +type DbInfo string // nolint: stylecheck - StreamSize() (int, error) -} +func (i DbInfo) String() string { return string(i) } -// VectoredDatabase scan the target data that consists of a list of non-contiguous blocks -// that are available all at once. -type VectoredDatabase interface { - Database - VectoredScanner - VectoredMatcher -} +var regexDBInfo = regexp.MustCompile(`^Version: (\d+\.\d+\.\d+) Features: ([\w\s]+)? Mode: (\w+)$`) -const infoMatches = 4 +const dbInfoMatches = 4 -var regexInfo = regexp.MustCompile(`^Version: (\d+\.\d+\.\d+) Features: ([\w\s]+)? Mode: (\w+)$`) +// Parse the version and platform information. +func (i DbInfo) Parse() (version, features, mode string, err error) { + matched := regexDBInfo.FindStringSubmatch(string(i)) -// DbInfo identify the version and platform information for the supplied database. -type DbInfo string // nolint: stylecheck + if len(matched) != dbInfoMatches { + err = fmt.Errorf("database info `%s`, %w", i, ErrInvalid) + } else { + version = matched[1] + features = matched[2] + mode = matched[3] + } -func (i DbInfo) String() string { return string(i) } + return +} // Version is the version for the supplied database. func (i DbInfo) Version() (string, error) { - matched := regexInfo.FindStringSubmatch(string(i)) + version, _, _, err := i.Parse() - if len(matched) != infoMatches { - return "", fmt.Errorf("database info, %w", ErrInvalid) - } - - return matched[1], nil + return version, err } // Mode is the scanning mode for the supplied database. func (i DbInfo) Mode() (ModeFlag, error) { - matched := regexInfo.FindStringSubmatch(string(i)) - - if len(matched) != infoMatches { - return 0, fmt.Errorf("database info, %w", ErrInvalid) + _, _, mode, err := i.Parse() + if err != nil { + return 0, err } - return ParseModeFlag(matched[3]) + return ParseModeFlag(mode) } // Version identify this release version. The return version is a string @@ -123,7 +105,7 @@ func Version() string { return hs.Version() } func ValidPlatform() error { return hs.ValidPlatform() } // nolint: wrapcheck type database interface { - Db() hs.Database + c() hs.Database } type baseDatabase struct { @@ -184,7 +166,7 @@ func SerializedDatabaseInfo(data []byte) (DbInfo, error) { return DbInfo(i), err } -func (d *baseDatabase) Db() hs.Database { return d.db } // nolint: stylecheck +func (d *baseDatabase) c() hs.Database { return d.db } func (d *baseDatabase) Size() (int, error) { return hs.DatabaseSize(d.db) } // nolint: wrapcheck @@ -202,29 +184,3 @@ func (d *baseDatabase) Close() error { return hs.FreeDatabase(d.db) } // nolint: func (d *baseDatabase) Marshal() ([]byte, error) { return hs.SerializeDatabase(d.db) } // nolint: wrapcheck func (d *baseDatabase) Unmarshal(data []byte) error { return hs.DeserializeDatabaseAt(data, d.db) } // nolint: wrapcheck - -type blockDatabase struct { - *blockMatcher -} - -func newBlockDatabase(db hs.Database) *blockDatabase { - return &blockDatabase{newBlockMatcher(newBlockScanner(newBaseDatabase(db)))} -} - -type streamDatabase struct { - *streamMatcher -} - -func newStreamDatabase(db hs.Database) *streamDatabase { - return &streamDatabase{newStreamMatcher(newStreamScanner(newBaseDatabase(db)))} -} - -func (db *streamDatabase) StreamSize() (int, error) { return hs.StreamSize(db.db) } // nolint: wrapcheck - -type vectoredDatabase struct { - *vectoredMatcher -} - -func newVectoredDatabase(db hs.Database) *vectoredDatabase { - return &vectoredDatabase{newVectoredMatcher(newVectoredScanner(newBaseDatabase(db)))} -} diff --git a/hyperscan/common_test.go b/hyperscan/common_test.go index e445981..a2c00d6 100644 --- a/hyperscan/common_test.go +++ b/hyperscan/common_test.go @@ -2,7 +2,6 @@ package hyperscan_test import ( - "regexp" "testing" . "github.com/smartystreets/goconvey/convey" @@ -10,8 +9,6 @@ import ( "github.com/flier/gohs/hyperscan" ) -var regexInfo = regexp.MustCompile(`^Version: (\d+\.\d+\.\d+) Features: ([\w\s]+)? Mode: (\w+)$`) - func TestBaseDatabase(t *testing.T) { Convey("Given a block database", t, func() { So(hyperscan.ValidPlatform(), ShouldBeNil) @@ -34,7 +31,8 @@ func TestBaseDatabase(t *testing.T) { So(err, ShouldBeNil) So(info, ShouldNotBeNil) - So(regexInfo.MatchString(info.String()), ShouldBeTrue) + _, _, _, err = info.Parse() + So(err, ShouldBeNil) Convey("Then get version", func() { ver, err := info.Version() @@ -70,7 +68,8 @@ func TestBaseDatabase(t *testing.T) { So(err, ShouldBeNil) So(info, ShouldNotBeNil) - So(regexInfo.MatchString(info.String()), ShouldBeTrue) + _, _, _, err = info.Parse() + So(err, ShouldBeNil) }) Convey("Then deserialize database", func() { @@ -85,7 +84,8 @@ func TestBaseDatabase(t *testing.T) { So(err, ShouldBeNil) So(info, ShouldNotBeNil) - So(regexInfo.MatchString(info.String()), ShouldBeTrue) + _, _, _, err = info.Parse() + So(err, ShouldBeNil) Convey("Then get version", func() { ver, err := info.Version() @@ -107,7 +107,8 @@ func TestBaseDatabase(t *testing.T) { So(err, ShouldBeNil) So(info, ShouldNotBeNil) - So(regexInfo.MatchString(info.String()), ShouldBeTrue) + _, _, _, err = info.Parse() + So(err, ShouldBeNil) Convey("Then get version", func() { ver, err := info.Version() @@ -136,7 +137,8 @@ func TestBlockDatabase(t *testing.T) { So(err, ShouldBeNil) So(info, ShouldNotBeNil) - So(regexInfo.MatchString(info.String()), ShouldBeTrue) + _, _, _, err = info.Parse() + So(err, ShouldBeNil) Convey("Then get mode", func() { mode, err := info.Mode() @@ -163,7 +165,8 @@ func TestVectoredDatabase(t *testing.T) { So(err, ShouldBeNil) So(info, ShouldNotBeNil) - So(regexInfo.MatchString(info.String()), ShouldBeTrue) + _, _, _, err = info.Parse() + So(err, ShouldBeNil) Convey("Then get mode", func() { mode, err := info.Mode() @@ -190,7 +193,8 @@ func TestStreamDatabase(t *testing.T) { So(err, ShouldBeNil) So(info, ShouldNotBeNil) - So(regexInfo.MatchString(info.String()), ShouldBeTrue) + _, _, _, err = info.Parse() + So(err, ShouldBeNil) Convey("Then get mode", func() { mode, err := info.Mode() diff --git a/hyperscan/scratch.go b/hyperscan/scratch.go index 9293828..c7c72d1 100644 --- a/hyperscan/scratch.go +++ b/hyperscan/scratch.go @@ -15,7 +15,7 @@ type Scratch struct { // This is required for runtime use, and one scratch space per thread, // or concurrent caller, is required. func NewScratch(db Database) (*Scratch, error) { - s, err := hs.AllocScratch(db.(database).Db()) + s, err := hs.AllocScratch(db.(database).c()) if err != nil { return nil, err // nolint: wrapcheck } @@ -43,7 +43,7 @@ func (s *Scratch) Size() (int, error) { return hs.ScratchSize(s.s) } // nolint: // Realloc reallocate the scratch for another database. func (s *Scratch) Realloc(db Database) error { - return hs.ReallocScratch(db.(database).Db(), &s.s) // nolint: wrapcheck + return hs.ReallocScratch(db.(database).c(), &s.s) // nolint: wrapcheck } // Clone allocate a scratch space that is a clone of an existing scratch space. diff --git a/hyperscan/stream.go b/hyperscan/stream.go index ccb4c96..bb674f5 100644 --- a/hyperscan/stream.go +++ b/hyperscan/stream.go @@ -63,6 +63,28 @@ type StreamCompressor interface { handler MatchHandler, context interface{}) (Stream, error) } +// StreamDatabase scan the target data to be scanned is a continuous stream, +// not all of which is available at once; +// blocks of data are scanned in sequence and matches may span multiple blocks in a stream. +type StreamDatabase interface { + Database + StreamScanner + StreamMatcher + StreamCompressor + + StreamSize() (int, error) +} + +type streamDatabase struct { + *streamMatcher +} + +func newStreamDatabase(db hs.Database) *streamDatabase { + return &streamDatabase{newStreamMatcher(newStreamScanner(newBaseDatabase(db)))} +} + +func (db *streamDatabase) StreamSize() (int, error) { return hs.StreamSize(db.db) } // nolint: wrapcheck + const bufSize = 4096 type stream struct { diff --git a/hyperscan/vectored.go b/hyperscan/vectored.go index e5f6453..75f3c73 100644 --- a/hyperscan/vectored.go +++ b/hyperscan/vectored.go @@ -10,6 +10,22 @@ type VectoredScanner interface { // VectoredMatcher implements regular expression search. type VectoredMatcher interface{} +// VectoredDatabase scan the target data that consists of a list of non-contiguous blocks +// that are available all at once. +type VectoredDatabase interface { + Database + VectoredScanner + VectoredMatcher +} + +type vectoredDatabase struct { + *vectoredMatcher +} + +func newVectoredDatabase(db hs.Database) *vectoredDatabase { + return &vectoredDatabase{newVectoredMatcher(newVectoredScanner(newBaseDatabase(db)))} +} + type vectoredScanner struct { *baseDatabase } diff --git a/internal/ch/allocator.go b/internal/ch/allocator.go index 928d5f4..92808d2 100644 --- a/internal/ch/allocator.go +++ b/internal/ch/allocator.go @@ -116,6 +116,10 @@ func SetAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { return } +func ClearAllocator() error { + return SetAllocator(nil, nil) +} + //export chDbAlloc func chDbAlloc(size C.size_t) unsafe.Pointer { return dbAllocator.Alloc(uint(size)) @@ -137,7 +141,7 @@ func SetDatabaseAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { ret = C.ch_set_database_allocator(nil, nil) } else { dbAllocator = Allocator{allocFunc, freeFunc} - ret = C.ch_set_database_allocator(C.ch_alloc_t(C.chDefaultAlloc), C.ch_free_t(C.chDefaultFree)) + ret = C.ch_set_database_allocator(C.ch_alloc_t(C.chDbAlloc), C.ch_free_t(C.chDbFree)) } if ret != C.CH_SUCCESS { @@ -147,6 +151,10 @@ func SetDatabaseAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { return } +func ClearDatabaseAllocator() error { + return SetDatabaseAllocator(nil, nil) +} + //export chMiscAlloc func chMiscAlloc(size C.size_t) unsafe.Pointer { return miscAllocator.Alloc(uint(size)) @@ -167,7 +175,7 @@ func SetMiscAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { ret = C.ch_set_misc_allocator(nil, nil) } else { miscAllocator = Allocator{allocFunc, freeFunc} - ret = C.ch_set_misc_allocator(C.ch_alloc_t(C.chDefaultAlloc), C.ch_free_t(C.chDefaultFree)) + ret = C.ch_set_misc_allocator(C.ch_alloc_t(C.chMiscAlloc), C.ch_free_t(C.chMiscFree)) } if ret != C.CH_SUCCESS { @@ -177,6 +185,10 @@ func SetMiscAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { return } +func ClearMiscAllocator() error { + return SetMiscAllocator(nil, nil) +} + //export chScratchAlloc func chScratchAlloc(size C.size_t) unsafe.Pointer { return scratchAllocator.Alloc(uint(size)) @@ -189,7 +201,7 @@ func chScratchFree(ptr unsafe.Pointer) { // Set the allocate and free functions used by Chimera for allocating memory // for scratch space by @ref ch_alloc_scratch() and @ref ch_clone_scratch(). -func SetscratchAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { +func SetScratchAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { var ret C.ch_error_t if allocFunc == nil || freeFunc == nil { @@ -197,7 +209,7 @@ func SetscratchAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { ret = C.ch_set_scratch_allocator(nil, nil) } else { scratchAllocator = Allocator{allocFunc, freeFunc} - ret = C.ch_set_scratch_allocator(C.ch_alloc_t(C.chDefaultAlloc), C.ch_free_t(C.chDefaultFree)) + ret = C.ch_set_scratch_allocator(C.ch_alloc_t(C.chScratchAlloc), C.ch_free_t(C.chScratchFree)) } if ret != C.CH_SUCCESS { @@ -206,3 +218,7 @@ func SetscratchAllocator(allocFunc AllocFunc, freeFunc FreeFunc) (err error) { return } + +func ClearScratchAllocator() error { + return SetScratchAllocator(nil, nil) +} diff --git a/internal/ch/allocator_test.go b/internal/ch/allocator_test.go new file mode 100644 index 0000000..e04c80a --- /dev/null +++ b/internal/ch/allocator_test.go @@ -0,0 +1,106 @@ +package ch_test + +import ( + "testing" + "unsafe" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/internal/ch" + "github.com/flier/gohs/internal/hs" +) + +type testAllocator struct { + memoryUsed int + memoryFreed []unsafe.Pointer +} + +func (a *testAllocator) alloc(size uint) unsafe.Pointer { + a.memoryUsed += int(size) + + return ch.AlignedAlloc(size) +} + +func (a *testAllocator) free(ptr unsafe.Pointer) { + a.memoryFreed = append(a.memoryFreed, ptr) + + ch.AlignedFree(ptr) +} + +//nolint:funlen +func TestAllocator(t *testing.T) { + Convey("Given the host platform", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + a := &testAllocator{} + + Convey("Then create a database with allocator", func() { + So(ch.SetDatabaseAllocator(a.alloc, a.free), ShouldBeNil) + + db, err := ch.Compile("test", 0, ch.Groups, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Then get the database info with allocator", func() { + So(ch.SetMiscAllocator(a.alloc, a.free), ShouldBeNil) + + s, err := ch.DatabaseInfo(db) + So(err, ShouldBeNil) + So(s, ShouldNotBeEmpty) + + So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, 12) + + So(ch.ClearMiscAllocator(), ShouldBeNil) + }) + + Convey("Get the database size", func() { + size, err := ch.DatabaseSize(db) + + So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, size) + So(err, ShouldBeNil) + }) + + Convey("Then create a scratch with allocator", func() { + So(ch.SetScratchAllocator(a.alloc, a.free), ShouldBeNil) + + a.memoryUsed = 0 + + s, err := ch.AllocScratch(db) + + So(s, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the scratch size", func() { + size, err := ch.ScratchSize(s) + + So(a.memoryUsed, ShouldBeGreaterThanOrEqualTo, size) + So(err, ShouldBeNil) + }) + + Convey("Then free scratch with allocator", func() { + a.memoryFreed = nil + + So(ch.FreeScratch(s), ShouldBeNil) + + So(a.memoryFreed[len(a.memoryFreed)-1], ShouldEqual, unsafe.Pointer(s)) + + So(ch.ClearScratchAllocator(), ShouldBeNil) + }) + }) + + Convey("Then free database with allocator", func() { + a.memoryFreed = nil + + So(ch.FreeDatabase(db), ShouldBeNil) + + So(a.memoryFreed, ShouldResemble, []unsafe.Pointer{unsafe.Pointer(db)}) + + So(ch.ClearDatabaseAllocator(), ShouldBeNil) + }) + }) + }) +} diff --git a/internal/ch/common.go b/internal/ch/common.go index 4b65106..730fc2c 100644 --- a/internal/ch/common.go +++ b/internal/ch/common.go @@ -1,10 +1,12 @@ package ch +import ( + "unsafe" +) + // #include import "C" -import "unsafe" - type Database *C.ch_database_t func FreeDatabase(db Database) (err error) { diff --git a/internal/ch/compile.go b/internal/ch/compile.go index 1e296a9..b4ba848 100644 --- a/internal/ch/compile.go +++ b/internal/ch/compile.go @@ -18,9 +18,6 @@ type Expression string func (e Expression) String() string { return string(e) } -// Patterns is a set of matching patterns. -type Patterns []*Pattern - // Pattern is a matching pattern. // nolint: golint,revive,stylecheck type Pattern struct { @@ -29,18 +26,6 @@ type Pattern struct { Id int // The ID number to be associated with the corresponding pattern } -func (p *Pattern) String() string { - var b strings.Builder - - if p.Id > 0 { - fmt.Fprintf(&b, "%d:", p.Id) - } - - fmt.Fprintf(&b, "/%s/%s", p.Expression, p.Flags) - - return b.String() -} - // A type containing error details that is returned by the compile calls on failure. // // The caller may inspect the values returned in this type to determine the cause of failure. @@ -133,8 +118,18 @@ func Compile(expression string, flags CompileFlag, mode CompileMode, info *hs.Pl return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) } +type Patterns interface { + Patterns() []*Pattern +} + +// The multiple regular expression compiler. +func CompileMulti(p Patterns, mode CompileMode, info *hs.PlatformInfo) (Database, error) { + return CompileExtMulti(p, mode, info, 0, 0) +} + // The multiple regular expression compiler. -func CompileMulti(patterns []*Pattern, mode CompileMode, info *hs.PlatformInfo) (Database, error) { +func CompileExtMulti(p Patterns, mode CompileMode, info *hs.PlatformInfo, + matchLimit, matchLimitRecursion uint) (Database, error) { var db *C.ch_database_t var err *C.ch_compile_error_t var platform *C.hs_platform_info_t @@ -143,6 +138,8 @@ func CompileMulti(patterns []*Pattern, mode CompileMode, info *hs.PlatformInfo) platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) } + patterns := p.Patterns() + cexprs := (**C.char)(C.calloc(C.size_t(len(patterns)), C.size_t(unsafe.Sizeof(uintptr(0))))) exprs := (*[1 << 30]*C.char)(unsafe.Pointer(cexprs))[:len(patterns):len(patterns)] @@ -158,7 +155,8 @@ func CompileMulti(patterns []*Pattern, mode CompileMode, info *hs.PlatformInfo) ids[i] = C.uint(pattern.Id) } - ret := C.ch_compile_multi(cexprs, cflags, cids, C.uint(len(patterns)), C.uint(mode), platform, &db, &err) + ret := C.ch_compile_ext_multi(cexprs, cflags, cids, C.uint(len(patterns)), C.uint(mode), + C.ulong(matchLimit), C.ulong(matchLimitRecursion), platform, &db, &err) for _, expr := range exprs { C.free(unsafe.Pointer(expr)) diff --git a/internal/ch/compile_test.go b/internal/ch/compile_test.go new file mode 100644 index 0000000..500b8f5 --- /dev/null +++ b/internal/ch/compile_test.go @@ -0,0 +1,67 @@ +package ch_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/chimera" + "github.com/flier/gohs/internal/ch" + "github.com/flier/gohs/internal/hs" +) + +func TestCompileAPI(t *testing.T) { + Convey("Given a host platform", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Compile an empty expression", func() { + db, err := ch.Compile("", 0, ch.Groups, platform) + + So(err, ShouldBeNil) + So(db, ShouldNotBeNil) + + So(ch.FreeDatabase(db), ShouldBeNil) + }) + + Convey("Compile multi expressions", func() { + db, err := ch.CompileMulti(chimera.Patterns{ + chimera.NewPattern(`^\w+`, 0), + chimera.NewPattern(`\d+`, 0), + chimera.NewPattern(`\s+`, 0), + }, ch.Groups, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the database info", func() { + _, err := ch.DatabaseInfo(db) + + So(err, ShouldBeNil) + }) + + So(ch.FreeDatabase(db), ShouldBeNil) + }) + + Convey("Compile multi expressions with extension", func() { + db, err := ch.CompileExtMulti(chimera.Patterns{ + chimera.NewPattern(`^\w+`, 0), + chimera.NewPattern(`\d+`, 0), + chimera.NewPattern(`\s+`, 0), + }, ch.Groups, platform, 1, 1) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the database info", func() { + _, err := ch.DatabaseInfo(db) + + So(err, ShouldBeNil) + }) + + So(ch.FreeDatabase(db), ShouldBeNil) + }) + }) +} diff --git a/internal/ch/runtime.go b/internal/ch/runtime.go index 8e4c390..b6f70f0 100644 --- a/internal/ch/runtime.go +++ b/internal/ch/runtime.go @@ -1,6 +1,7 @@ package ch import ( + "fmt" "reflect" "runtime" "unsafe" @@ -28,40 +29,6 @@ extern ch_callback_t errorEventCallback(ch_error_event_t error_type, */ import "C" -// A Chimera scratch space. -type Scratch *C.ch_scratch_t - -// Allocate a scratch space that is a clone of an existing scratch space. -func CloneScratch(scratch Scratch) (Scratch, error) { - var clone *C.ch_scratch_t - - if ret := C.ch_clone_scratch(scratch, &clone); ret != C.HS_SUCCESS { - return nil, Error(ret) - } - - return clone, nil -} - -// Provides the size of the given scratch space. -func ScratchSize(scratch Scratch) (int, error) { - var size C.size_t - - if ret := C.ch_scratch_size(scratch, &size); ret != C.HS_SUCCESS { - return 0, Error(ret) - } - - return int(size), nil -} - -// Free a scratch block previously allocated by @ref ch_alloc_scratch() or @ref ch_clone_scratch(). -func FreeScratch(scratch Scratch) error { - if ret := C.ch_free_scratch(scratch); ret != C.HS_SUCCESS { - return Error(ret) - } - - return nil -} - // Callback return value used to tell the Chimera matcher what to do after processing this match. type Callback C.ch_callback_t @@ -80,15 +47,27 @@ type Capture struct { // Definition of the match event callback function type. type MatchEventHandler func(id uint, from, to uint64, flags uint, captured []*Capture, context interface{}) Callback +// Type used to differentiate the errors raised with the `ErrorEventHandler` callback. type ErrorEvent C.ch_error_event_t const ( // PCRE hits its match limit and reports PCRE_ERROR_MATCHLIMIT. - MatchLimit ErrorEvent = C.CH_ERROR_MATCHLIMIT + ErrMatchLimit ErrorEvent = C.CH_ERROR_MATCHLIMIT // PCRE hits its recursion limit and reports PCRE_ERROR_RECURSIONLIMIT. - RecursionLimit ErrorEvent = C.CH_ERROR_RECURSIONLIMIT + ErrRecursionLimit ErrorEvent = C.CH_ERROR_RECURSIONLIMIT ) +func (e ErrorEvent) Error() string { + switch e { + case ErrMatchLimit: + return "PCRE hits its match limit." + case ErrRecursionLimit: + return "PCRE hits its recursion limit." + } + + return fmt.Sprintf("ErrorEvent(%d)", int(C.ch_error_event_t(e))) +} + // Definition of the Chimera error event callback function type. type ErrorEventHandler func(event ErrorEvent, id uint, info, context interface{}) Callback @@ -126,10 +105,10 @@ func errorEventCallback(evt C.ch_error_event_t, id C.uint, info, data unsafe.Poi } // ScanFlag represents a scan flag. -type ScanFlag C.uint +type ScanFlag uint // The block regular expression scanner. -func hsScan(db Database, data []byte, flags ScanFlag, scratch Scratch, +func Scan(db Database, data []byte, flags ScanFlag, scratch Scratch, onEvent MatchEventHandler, onError ErrorEventHandler, context interface{}) error { if data == nil { return ErrInvalid @@ -158,3 +137,29 @@ func hsScan(db Database, data []byte, flags ScanFlag, scratch Scratch, return nil } + +type MatchEvent struct { + ID uint + From, To uint64 + Flag ScanFlag + Captured []*Capture +} + +type MatchRecorder struct { + Events []MatchEvent + Err error +} + +func (h *MatchRecorder) Matched() bool { return len(h.Events) > 0 } + +func (h *MatchRecorder) OnMatch(id uint, from, to uint64, flags uint, captured []*Capture, ctx interface{}) Callback { + h.Events = append(h.Events, MatchEvent{id, from, to, ScanFlag(flags), captured}) + + return Continue +} + +func (h *MatchRecorder) OnError(evt ErrorEvent, id uint, info, ctx interface{}) Callback { + h.Err = evt + + return Terminate +} diff --git a/internal/ch/runtime_test.go b/internal/ch/runtime_test.go new file mode 100644 index 0000000..a9071ae --- /dev/null +++ b/internal/ch/runtime_test.go @@ -0,0 +1,65 @@ +package ch_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/internal/ch" + "github.com/flier/gohs/internal/hs" +) + +func TestBlockScan(t *testing.T) { + Convey("Given a block database", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + db, err := ch.Compile("test", 0, ch.Groups, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + s, err := ch.AllocScratch(db) + + So(s, ShouldNotBeNil) + So(err, ShouldBeNil) + + h := &ch.MatchRecorder{} + + Convey("Scan block with pattern", func() { + So(ch.Scan(db, []byte("abctestdef"), 0, s, h.OnMatch, h.OnError, nil), ShouldBeNil) + So(h.Events, ShouldResemble, []ch.MatchEvent{{0, 3, 7, 0, []*ch.Capture{{3, 7}}}}) + }) + + Convey("Scan block without pattern", func() { + So(ch.Scan(db, []byte("abcdef"), 0, s, h.OnMatch, h.OnError, nil), ShouldBeNil) + So(h.Events, ShouldBeEmpty) + }) + + Convey("Scan block with multi pattern", func() { + So(ch.Scan(db, []byte("abctestdeftest"), 0, s, h.OnMatch, h.OnError, nil), ShouldBeNil) + So(h.Events, ShouldResemble, []ch.MatchEvent{ + {0, 3, 7, 0, []*ch.Capture{{3, 7}}}, + {0, 10, 14, 0, []*ch.Capture{{10, 14}}}, + }) + }) + + Convey("Scan block with multi pattern but terminated", func() { + onMatch := func(id uint, from, to uint64, flags uint, captured []*ch.Capture, context interface{}) ch.Callback { + return ch.Terminate + } + + So(ch.Scan(db, []byte("abctestdeftest"), 0, s, onMatch, h.OnError, nil), ShouldEqual, ch.ErrScanTerminated) + }) + + Convey("Scan empty buffers", func() { + So(ch.Scan(db, nil, 0, s, h.OnMatch, h.OnError, nil), ShouldEqual, ch.ErrInvalid) + So(ch.Scan(db, []byte(""), 0, s, h.OnMatch, h.OnError, nil), ShouldBeNil) + }) + + So(ch.FreeScratch(s), ShouldBeNil) + So(ch.FreeDatabase(db), ShouldBeNil) + }) +} diff --git a/internal/ch/scratch.go b/internal/ch/scratch.go new file mode 100644 index 0000000..e08f268 --- /dev/null +++ b/internal/ch/scratch.go @@ -0,0 +1,52 @@ +package ch + +// #include +import "C" + +type Scratch *C.ch_scratch_t + +func AllocScratch(db Database) (Scratch, error) { + var scratch *C.ch_scratch_t + + if ret := C.ch_alloc_scratch(db, &scratch); ret != C.CH_SUCCESS { + return nil, Error(ret) + } + + return scratch, nil +} + +func ReallocScratch(db Database, scratch *Scratch) error { + if ret := C.ch_alloc_scratch(db, (**C.struct_ch_scratch)(scratch)); ret != C.CH_SUCCESS { + return Error(ret) + } + + return nil +} + +func CloneScratch(scratch Scratch) (Scratch, error) { + var clone *C.ch_scratch_t + + if ret := C.ch_clone_scratch(scratch, &clone); ret != C.CH_SUCCESS { + return nil, Error(ret) + } + + return clone, nil +} + +func ScratchSize(scratch Scratch) (int, error) { + var size C.size_t + + if ret := C.ch_scratch_size(scratch, &size); ret != C.CH_SUCCESS { + return 0, Error(ret) + } + + return int(size), nil +} + +func FreeScratch(scratch Scratch) error { + if ret := C.ch_free_scratch(scratch); ret != C.CH_SUCCESS { + return Error(ret) + } + + return nil +} diff --git a/internal/ch/scratch_test.go b/internal/ch/scratch_test.go new file mode 100644 index 0000000..40b18cd --- /dev/null +++ b/internal/ch/scratch_test.go @@ -0,0 +1,77 @@ +package ch_test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/hyperscan" + "github.com/flier/gohs/internal/ch" + "github.com/flier/gohs/internal/hs" +) + +//nolint:funlen +func TestScratch(t *testing.T) { + Convey("Given a block database", t, func() { + platform, err := hs.PopulatePlatform() + + So(platform, ShouldNotBeNil) + So(err, ShouldBeNil) + + db, err := ch.Compile("test", 0, ch.Groups, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Allocate a scratch", func() { + s, err := ch.AllocScratch(db) + + So(s, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Get the scratch size", func() { + size, err := ch.ScratchSize(s) + + So(size, ShouldBeGreaterThan, 1024) + So(size, ShouldBeLessThan, 4096) + So(err, ShouldBeNil) + + Convey("Clone the scratch", func() { + s2, err := ch.CloneScratch(s) + + So(s2, ShouldNotBeNil) + So(err, ShouldBeNil) + + Convey("Cloned scrash should have same size", func() { + size2, err := ch.ScratchSize(s2) + + So(size2, ShouldEqual, size) + So(err, ShouldBeNil) + }) + + So(ch.FreeScratch(s2), ShouldBeNil) + }) + + Convey("Reallocate the scratch with another database", func() { + db2, err := ch.Compile(hyperscan.EmailAddress, 0, ch.Groups, platform) + + So(db, ShouldNotBeNil) + So(err, ShouldBeNil) + + So(ch.ReallocScratch(db2, &s), ShouldBeNil) + + size2, err := ch.ScratchSize(s) + + So(size2, ShouldBeGreaterThan, size) + So(err, ShouldBeNil) + + So(ch.FreeDatabase(db2), ShouldBeNil) + }) + }) + + So(ch.FreeScratch(s), ShouldBeNil) + }) + + So(ch.FreeDatabase(db), ShouldBeNil) + }) +} From 47d50184998a97baffc91bc6a3772a9b7a8baf50 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Mon, 27 Sep 2021 19:47:28 +0800 Subject: [PATCH 09/34] improve examples --- chimera/block.go | 2 +- chimera/common.go | 13 +--- chimera/common_test.go | 7 +- chimera/compile.go | 18 +++-- chimera/compile_test.go | 2 +- chimera/example_runtime_test.go | 2 +- chimera/runtime.go | 18 ++--- chimera/runtime_test.go | 2 +- examples/pcapscan/main.go | 4 +- examples/simplegrep/main.go | 131 ++++++++++++++++++++++---------- hyperscan/common.go | 35 +++++---- hyperscan/compile.go | 9 ++- internal/ch/runtime.go | 13 ++-- internal/ch/runtime_test.go | 6 +- 14 files changed, 165 insertions(+), 97 deletions(-) diff --git a/chimera/block.go b/chimera/block.go index e4ee01c..91a5fdd 100644 --- a/chimera/block.go +++ b/chimera/block.go @@ -116,7 +116,7 @@ func (m *blockMatcher) OnMatch(id uint, from, to uint64, flags uint, return Continue } - if m.n < len(m.Events) { + if m.n <= len(m.Events) { m.Events = m.Events[:m.n] return Terminate diff --git a/chimera/common.go b/chimera/common.go index c96d493..6f1a691 100644 --- a/chimera/common.go +++ b/chimera/common.go @@ -37,14 +37,9 @@ const ( ErrScratchInUse Error = ch.ErrScratchInUse ) -// ModeFlag represents the compile mode flags. -type ModeFlag = hyperscan.ModeFlag - -// BlockMode for the block scan (non-streaming) database. -const BlockMode = hyperscan.BlockMode - // DbInfo identify the version and platform information for the supplied database. -type DbInfo string // nolint: stylecheck +// nolint: stylecheck +type DbInfo string // parse `Chimera Version: 5.4.0 Features: AVX2 Mode: BLOCK`. var regexDBInfo = regexp.MustCompile(`^Chimera Version: (\d+\.\d+\.\d+) Features: ([\w\s]+)? Mode: (\w+)$`) @@ -76,7 +71,7 @@ func (i DbInfo) Version() (string, error) { } // Mode is the scanning mode for the supplied database. -func (i DbInfo) Mode() (ModeFlag, error) { +func (i DbInfo) Mode() (hyperscan.ModeFlag, error) { _, _, mode, err := i.Parse() if err != nil { return 0, err @@ -85,7 +80,7 @@ func (i DbInfo) Mode() (ModeFlag, error) { return hyperscan.ParseModeFlag(mode) // nolint: wrapcheck } -// Database is an immutable database that can be used by the Hyperscan scanning API. +// Database is an immutable database that can be used by the Chimera scanning API. type Database interface { // Provides information about a database. Info() (DbInfo, error) diff --git a/chimera/common_test.go b/chimera/common_test.go index 99fd7c0..fc0ae96 100644 --- a/chimera/common_test.go +++ b/chimera/common_test.go @@ -6,6 +6,7 @@ import ( . "github.com/smartystreets/goconvey/convey" "github.com/flier/gohs/chimera" + "github.com/flier/gohs/hyperscan" ) func TestChimera(t *testing.T) { @@ -16,7 +17,7 @@ func TestChimera(t *testing.T) { func TestBaseDatabase(t *testing.T) { Convey("Given a block database", t, func() { - So(chimera.ValidPlatform(), ShouldBeNil) + So(hyperscan.ValidPlatform(), ShouldBeNil) bdb, err := chimera.NewBlockDatabase(&chimera.Pattern{Expression: "test"}) @@ -50,7 +51,7 @@ func TestBaseDatabase(t *testing.T) { mode, err := info.Mode() So(err, ShouldBeNil) - So(mode, ShouldEqual, chimera.BlockMode) + So(mode, ShouldEqual, hyperscan.BlockMode) }) }) @@ -78,7 +79,7 @@ func TestBlockDatabase(t *testing.T) { mode, err := info.Mode() So(err, ShouldBeNil) - So(mode, ShouldEqual, chimera.BlockMode) + So(mode, ShouldEqual, hyperscan.BlockMode) }) }) diff --git a/chimera/compile.go b/chimera/compile.go index 7a52170..8ce4a8b 100644 --- a/chimera/compile.go +++ b/chimera/compile.go @@ -5,6 +5,7 @@ import ( "runtime" "strconv" + "github.com/flier/gohs/hyperscan" "github.com/flier/gohs/internal/ch" "github.com/flier/gohs/internal/hs" ) @@ -67,31 +68,38 @@ const ( Groups CompileMode = ch.Groups ) +// Builder creates a database with the given mode and target platform. type Builder interface { + // Build the database with the given mode. Build(mode CompileMode) (Database, error) - ForPlatform(mode CompileMode, platform Platform) (Database, error) + // ForPlatform determine the target platform for the database + ForPlatform(mode CompileMode, platform hyperscan.Platform) (Database, error) } +// Build the database with the given mode. func (p *Pattern) Build(mode CompileMode) (Database, error) { return p.ForPlatform(mode, nil) } -func (p *Pattern) ForPlatform(mode CompileMode, platform Platform) (Database, error) { +// ForPlatform determine the target platform for the database. +func (p *Pattern) ForPlatform(mode CompileMode, platform hyperscan.Platform) (Database, error) { b := DatabaseBuilder{Patterns: Patterns{p}, Mode: mode, Platform: platform} return b.Build() } +// Build the database with the given mode. func (p Patterns) Build(mode CompileMode) (Database, error) { return p.ForPlatform(mode, nil) } -func (p Patterns) ForPlatform(mode CompileMode, platform Platform) (Database, error) { +// ForPlatform determine the target platform for the database. +func (p Patterns) ForPlatform(mode CompileMode, platform hyperscan.Platform) (Database, error) { b := DatabaseBuilder{Patterns: p, Mode: mode, Platform: platform} return b.Build() } -// DatabaseBuilder to help to build up a database. +// DatabaseBuilder creates a database that will be used to matching the patterns. type DatabaseBuilder struct { // Array of patterns to compile. Patterns @@ -101,7 +109,7 @@ type DatabaseBuilder struct { // If not nil, the platform structure is used to determine the target platform for the database. // If nil, a database suitable for running on the current host platform is produced. - Platform + hyperscan.Platform // A limit from pcre_extra on the amount of match function called in PCRE to limit backtracking that can take place. MatchLimit uint diff --git a/chimera/compile_test.go b/chimera/compile_test.go index b896e81..c62c784 100644 --- a/chimera/compile_test.go +++ b/chimera/compile_test.go @@ -65,7 +65,7 @@ func TestDatabaseBuilder(t *testing.T) { mode, err := info.Mode() So(err, ShouldBeNil) - So(mode, ShouldEqual, chimera.BlockMode) + So(mode, ShouldEqual, hyperscan.BlockMode) So(db.Close(), ShouldBeNil) }) diff --git a/chimera/example_runtime_test.go b/chimera/example_runtime_test.go index 542d24b..5d71be5 100644 --- a/chimera/example_runtime_test.go +++ b/chimera/example_runtime_test.go @@ -40,7 +40,7 @@ func ExampleBlockScanner() { var matches []Match - handler := chimera.MatchHandlerFunc(func(id uint, from, to uint64, flags uint, + handler := chimera.HandlerFunc(func(id uint, from, to uint64, flags uint, captured []*chimera.Capture, ctx interface{}) chimera.Callback { matches = append(matches, Match{from, to}) return chimera.Continue diff --git a/chimera/runtime.go b/chimera/runtime.go index 5165599..1b4b885 100644 --- a/chimera/runtime.go +++ b/chimera/runtime.go @@ -1,16 +1,9 @@ package chimera import ( - "github.com/flier/gohs/hyperscan" "github.com/flier/gohs/internal/ch" ) -// Platform is a type containing information on the target platform. -type Platform = hyperscan.Platform - -// ValidPlatform test the current system architecture. -var ValidPlatform = hyperscan.ValidPlatform - // Callback return value used to tell the Chimera matcher what to do after processing this match. type Callback = ch.Callback @@ -43,12 +36,17 @@ type Handler interface { OnError(event ErrorEvent, id uint, info, context interface{}) Callback } -type MatchHandlerFunc func(id uint, from, to uint64, flags uint, captured []*Capture, context interface{}) Callback +// HandlerFunc type is an adapter to allow the use of ordinary functions as Chimera handlers. +// If f is a function with the appropriate signature, HandlerFunc(f) is a Handler that calls f. +type HandlerFunc func(id uint, from, to uint64, flags uint, captured []*Capture, context interface{}) Callback -func (f MatchHandlerFunc) OnMatch(id uint, from, to uint64, flags uint, captured []*Capture, ctx interface{}) Callback { +// OnMatch will be invoked whenever a match is located in the target data during the execution of a scan. +func (f HandlerFunc) OnMatch(id uint, from, to uint64, flags uint, captured []*Capture, ctx interface{}) Callback { return f(id, from, to, flags, captured, ctx) } -func (f MatchHandlerFunc) OnError(event ErrorEvent, id uint, info, context interface{}) Callback { +// OnError will be invoked when an error event occurs during matching; +// this indicates that some matches for a given expression may not be reported. +func (f HandlerFunc) OnError(event ErrorEvent, id uint, info, context interface{}) Callback { return Terminate } diff --git a/chimera/runtime_test.go b/chimera/runtime_test.go index 2457db4..7801be9 100644 --- a/chimera/runtime_test.go +++ b/chimera/runtime_test.go @@ -35,7 +35,7 @@ func TestBlockScanner(t *testing.T) { return chimera.Continue } - err = bdb.Scan([]byte("abc123def456"), nil, chimera.MatchHandlerFunc(matched), nil) + err = bdb.Scan([]byte("abc123def456"), nil, chimera.HandlerFunc(matched), nil) So(err, ShouldBeNil) So(matches, ShouldResemble, [][]uint64{{3, 6}, {9, 12}}) diff --git a/examples/pcapscan/main.go b/examples/pcapscan/main.go index 5b20504..5ba4fb0 100644 --- a/examples/pcapscan/main.go +++ b/examples/pcapscan/main.go @@ -33,6 +33,7 @@ import ( "io/ioutil" "net" "os" + "path/filepath" "runtime/pprof" "strconv" "strings" @@ -412,7 +413,8 @@ func main() { flag.Parse() if flag.NArg() != 2 { - fmt.Fprintf(os.Stderr, "Usage: %s [-n repeats] \n", os.Args[0]) + _, prog := filepath.Split(os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [-n repeats] \n", prog) os.Exit(-1) } diff --git a/examples/simplegrep/main.go b/examples/simplegrep/main.go index 42954fd..cd3afc3 100644 --- a/examples/simplegrep/main.go +++ b/examples/simplegrep/main.go @@ -31,6 +31,10 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" + "sync" + "sync/atomic" + "time" "github.com/flier/gohs/hyperscan" ) @@ -46,17 +50,23 @@ func highlight(s string) string { return "\033[35m" + s + "\033[0m" } +type context struct { + *bytes.Buffer + filename string + data []byte +} + /** * This is the function that will be called for each match that occurs. @a ctx * is to allow you to have some application-specific state that you will get * access to for each match. In our simple example we're just going to use it * to pass in the pattern that was being searched for so we can print it out. */ -func eventHandler(id uint, from, to uint64, flags uint, context interface{}) error { - inputData := context.([]byte) +func eventHandler(id uint, from, to uint64, flags uint, data interface{}) error { + ctx, _ := data.(context) - start := bytes.LastIndexByte(inputData[:from], '\n') - end := int(to) + bytes.IndexByte(inputData[to:], '\n') + start := bytes.LastIndexByte(ctx.data[:from], '\n') + end := int(to) + bytes.IndexByte(ctx.data[to:], '\n') if start == -1 { start = 0 @@ -65,14 +75,14 @@ func eventHandler(id uint, from, to uint64, flags uint, context interface{}) err } if end == -1 { - end = len(inputData) + end = len(ctx.data) } + fmt.Fprintf(ctx, "%s", ctx.filename) if *flagByteOffset { - fmt.Printf("%d: ", start) + fmt.Fprintf(ctx, ":%d", start) } - - fmt.Printf("%s%s%s\n", inputData[start:from], theme(string(inputData[from:to])), inputData[to:end]) + fmt.Fprintf(ctx, "\t%s%s%s\n", ctx.data[start:from], theme(string(ctx.data[from:to])), ctx.data[to:end]) return nil } @@ -80,8 +90,9 @@ func eventHandler(id uint, from, to uint64, flags uint, context interface{}) err func main() { flag.Parse() - if flag.NArg() != 2 { - fmt.Fprintf(os.Stderr, "Usage: %s \n", os.Args[0]) + if flag.NArg() < 2 { + _, prog := filepath.Split(os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s \n", prog) os.Exit(-1) } @@ -94,7 +105,6 @@ func main() { } pattern := hyperscan.NewPattern(hyperscan.Expression(flag.Arg(0)), hyperscan.DotAll|hyperscan.SomLeftMost) - inputFN := flag.Arg(1) /* First, we attempt to compile the pattern provided on the command line. * We assume 'DOTALL' semantics, meaning that the '.' meta-character will @@ -110,46 +120,85 @@ func main() { defer database.Close() - /* Next, we read the input data file into a buffer. */ - inputData, err := ioutil.ReadFile(inputFN) - if err != nil { - os.Exit(-1) + scratchPool := sync.Pool{ + New: func() interface{} { + scratch, err := hyperscan.NewManagedScratch(database) + if err != nil { + fmt.Fprint(os.Stderr, "ERROR: Unable to allocate scratch space. Exiting.\n") + os.Exit(-1) + } + return scratch + }, } - - /* Finally, we issue a call to hs_scan, which will search the input buffer - * for the pattern represented in the bytecode. Note that in order to do - * this, scratch space needs to be allocated with the hs_alloc_scratch - * function. In typical usage, you would reuse this scratch space for many - * calls to hs_scan, but as we're only doing one, we'll be allocating it - * and deallocating it as soon as our matching is done. - * - * When matches occur, the specified callback function (eventHandler in - * this file) will be called. Note that although it is reminiscent of - * asynchronous APIs, Hyperscan operates synchronously: all matches will be - * found, and all callbacks issued, *before* hs_scan returns. - * - * In this example, we provide the input pattern as the context pointer so - * that the callback is able to print out the pattern that matched on each - * match event. - */ - scratch, err := hyperscan.NewScratch(database) - if err != nil { - fmt.Fprint(os.Stderr, "ERROR: Unable to allocate scratch space. Exiting.\n") - os.Exit(-1) + scratchAlloc := func() (*hyperscan.Scratch, func()) { + scratch, _ := scratchPool.Get().(*hyperscan.Scratch) + return scratch, func() { scratchPool.Put(scratch) } } - defer scratch.Free() + start := time.Now() + var files, size uint32 + var wg sync.WaitGroup - fmt.Printf("Scanning %d bytes with Hyperscan\n", len(inputData)) + for _, pattern := range os.Args[1:] { + filenames, err := filepath.Glob(pattern) + if err != nil { + fmt.Fprint(os.Stderr, "ERROR: Unable to list all files matching pattern. Exiting.\n") + os.Exit(-1) + } - if err := database.Scan(inputData, scratch, eventHandler, inputData); err != nil { - fmt.Fprint(os.Stderr, "ERROR: Unable to scan input buffer. Exiting.\n") - os.Exit(-1) + for _, filename := range filenames { + filename := filename + + go func() { + wg.Add(1) + defer wg.Done() + + scratch, release := scratchAlloc() + defer release() + + /* Next, we read the input data file into a buffer. */ + inputData, err := ioutil.ReadFile(filename) + if err != nil { + os.Exit(-1) + } + + atomic.AddUint32(&files, 1) + atomic.AddUint32(&size, uint32(len(inputData))) + + /* Finally, we issue a call to hs_scan, which will search the input buffer + * for the pattern represented in the bytecode. Note that in order to do + * this, scratch space needs to be allocated with the hs_alloc_scratch + * function. In typical usage, you would reuse this scratch space for many + * calls to hs_scan, but as we're only doing one, we'll be allocating it + * and deallocating it as soon as our matching is done. + * + * When matches occur, the specified callback function (eventHandler in + * this file) will be called. Note that although it is reminiscent of + * asynchronous APIs, Hyperscan operates synchronously: all matches will be + * found, and all callbacks issued, *before* hs_scan returns. + * + * In this example, we provide the input pattern as the context pointer so + * that the callback is able to print out the pattern that matched on each + * match event. + */ + + buf := new(bytes.Buffer) + if err := database.Scan(inputData, scratch, eventHandler, context{buf, filename, inputData}); err != nil { + fmt.Fprint(os.Stderr, "ERROR: Unable to scan input buffer. Exiting.\n") + os.Exit(-1) + } + fmt.Fprint(os.Stdout, buf.String()) + }() + } } + wg.Wait() + /* Scanning is complete, any matches have been handled, so now we just * clean up and exit. */ + fmt.Printf("Scanning %d bytes in %d files with Hyperscan in %s\n", size, files, time.Since(start)) + return } diff --git a/hyperscan/common.go b/hyperscan/common.go index 8f87914..e6760db 100644 --- a/hyperscan/common.go +++ b/hyperscan/common.go @@ -7,35 +7,39 @@ import ( "github.com/flier/gohs/internal/hs" ) -type HsError = hs.Error +// HsError is the type type for errors returned by Hyperscan functions. +type HsError = Error + +// Error is the type type for errors returned by Hyperscan functions. +type Error = hs.Error const ( // ErrSuccess is the error returned if the engine completed normally. - ErrSuccess HsError = hs.ErrSuccess + ErrSuccess Error = hs.ErrSuccess // ErrInvalid is the error returned if a parameter passed to this function was invalid. - ErrInvalid HsError = hs.ErrInvalid + ErrInvalid Error = hs.ErrInvalid // ErrNoMemory is the error returned if a memory allocation failed. - ErrNoMemory HsError = hs.ErrNoMemory + ErrNoMemory Error = hs.ErrNoMemory // ErrScanTerminated is the error returned if the engine was terminated by callback. - ErrScanTerminated HsError = hs.ErrScanTerminated + ErrScanTerminated Error = hs.ErrScanTerminated // ErrCompileError is the error returned if the pattern compiler failed. - ErrCompileError HsError = hs.ErrCompileError + ErrCompileError Error = hs.ErrCompileError // ErrDatabaseVersionError is the error returned if the given database was built for a different version of Hyperscan. - ErrDatabaseVersionError HsError = hs.ErrDatabaseVersionError + ErrDatabaseVersionError Error = hs.ErrDatabaseVersionError // ErrDatabasePlatformError is the error returned if the given database was built for a different platform. - ErrDatabasePlatformError HsError = hs.ErrDatabasePlatformError + ErrDatabasePlatformError Error = hs.ErrDatabasePlatformError // ErrDatabaseModeError is the error returned if the given database was built for a different mode of operation. - ErrDatabaseModeError HsError = hs.ErrDatabaseModeError + ErrDatabaseModeError Error = hs.ErrDatabaseModeError // ErrBadAlign is the error returned if a parameter passed to this function was not correctly aligned. - ErrBadAlign HsError = hs.ErrBadAlign + ErrBadAlign Error = hs.ErrBadAlign // ErrBadAlloc is the error returned if the memory allocator did not correctly return memory suitably aligned. - ErrBadAlloc HsError = hs.ErrBadAlloc + ErrBadAlloc Error = hs.ErrBadAlloc // ErrScratchInUse is the error returned if the scratch region was already in use. - ErrScratchInUse HsError = hs.ErrScratchInUse + ErrScratchInUse Error = hs.ErrScratchInUse // ErrArchError is the error returned if unsupported CPU architecture. - ErrArchError HsError = hs.ErrArchError + ErrArchError Error = hs.ErrArchError // ErrInsufficientSpace is the error returned if provided buffer was too small. - ErrInsufficientSpace HsError = hs.ErrInsufficientSpace + ErrInsufficientSpace Error = hs.ErrInsufficientSpace ) // Database is an immutable database that can be used by the Hyperscan scanning API. @@ -57,7 +61,8 @@ type Database interface { } // DbInfo identify the version and platform information for the supplied database. -type DbInfo string // nolint: stylecheck +// nolint: stylecheck +type DbInfo string func (i DbInfo) String() string { return string(i) } diff --git a/hyperscan/compile.go b/hyperscan/compile.go index 94afb91..2d7e7d5 100644 --- a/hyperscan/compile.go +++ b/hyperscan/compile.go @@ -98,31 +98,38 @@ func ParseModeFlag(s string) (ModeFlag, error) { return BlockMode, fmt.Errorf("database mode %s, %w", s, ErrInvalid) } +// Builder creates a database with the given mode and target platform. type Builder interface { + // Build the database with the given mode. Build(mode ModeFlag) (Database, error) + // ForPlatform determine the target platform for the database ForPlatform(mode ModeFlag, platform Platform) (Database, error) } +// Build the database with the given mode. func (p *Pattern) Build(mode ModeFlag) (Database, error) { return p.ForPlatform(mode, nil) } +// ForPlatform determine the target platform for the database. func (p *Pattern) ForPlatform(mode ModeFlag, platform Platform) (Database, error) { b := DatabaseBuilder{Patterns: Patterns{p}, Mode: mode, Platform: platform} return b.Build() } +// Build the database with the given mode. func (p Patterns) Build(mode ModeFlag) (Database, error) { return p.ForPlatform(mode, nil) } +// ForPlatform determine the target platform for the database. func (p Patterns) ForPlatform(mode ModeFlag, platform Platform) (Database, error) { b := DatabaseBuilder{Patterns: p, Mode: mode, Platform: platform} return b.Build() } -// DatabaseBuilder to help to build up a database. +// DatabaseBuilder creates a database that will be used to matching the patterns. type DatabaseBuilder struct { // Array of patterns to compile. Patterns diff --git a/internal/ch/runtime.go b/internal/ch/runtime.go index b6f70f0..e3b4d92 100644 --- a/internal/ch/runtime.go +++ b/internal/ch/runtime.go @@ -40,8 +40,9 @@ const ( // Capture representing a captured subexpression within a match. type Capture struct { - From uint64 // offset at which this capture group begins. - To uint64 // offset at which this capture group ends. + From uint64 // offset at which this capture group begins. + To uint64 // offset at which this capture group ends. + Bytes []byte // matches of the expression } // Definition of the match event callback function type. @@ -72,13 +73,15 @@ func (e ErrorEvent) Error() string { type ErrorEventHandler func(event ErrorEvent, id uint, info, context interface{}) Callback type eventContext struct { + data []byte onEvent MatchEventHandler onError ErrorEventHandler context interface{} } //export matchEventCallback -func matchEventCallback(id C.uint, from, to C.ulonglong, flags, size C.uint, cap *C.capture_t, data unsafe.Pointer) C.ch_callback_t { +func matchEventCallback(id C.uint, from, to C.ulonglong, flags, size C.uint, + cap *C.capture_t, data unsafe.Pointer) C.ch_callback_t { ctx, ok := handle.Handle(data).Value().(eventContext) if !ok { return C.CH_CALLBACK_TERMINATE @@ -87,7 +90,7 @@ func matchEventCallback(id C.uint, from, to C.ulonglong, flags, size C.uint, cap captured := make([]*Capture, size) for i, c := range (*[1 << 30]C.capture_t)(unsafe.Pointer(cap))[:size:size] { if c.flags == C.CH_CAPTURE_FLAG_ACTIVE { - captured[i] = &Capture{uint64(c.from), uint64(c.to)} + captured[i] = &Capture{uint64(c.from), uint64(c.to), ctx.data[c.from:c.to]} } } @@ -114,7 +117,7 @@ func Scan(db Database, data []byte, flags ScanFlag, scratch Scratch, return ErrInvalid } - h := handle.New(eventContext{onEvent, onError, context}) + h := handle.New(eventContext{data, onEvent, onError, context}) defer h.Delete() hdr := (*reflect.SliceHeader)(unsafe.Pointer(&data)) // FIXME: Zero-copy access to go data diff --git a/internal/ch/runtime_test.go b/internal/ch/runtime_test.go index a9071ae..7df4f12 100644 --- a/internal/ch/runtime_test.go +++ b/internal/ch/runtime_test.go @@ -30,7 +30,7 @@ func TestBlockScan(t *testing.T) { Convey("Scan block with pattern", func() { So(ch.Scan(db, []byte("abctestdef"), 0, s, h.OnMatch, h.OnError, nil), ShouldBeNil) - So(h.Events, ShouldResemble, []ch.MatchEvent{{0, 3, 7, 0, []*ch.Capture{{3, 7}}}}) + So(h.Events, ShouldResemble, []ch.MatchEvent{{0, 3, 7, 0, []*ch.Capture{{3, 7, []byte("test")}}}}) }) Convey("Scan block without pattern", func() { @@ -41,8 +41,8 @@ func TestBlockScan(t *testing.T) { Convey("Scan block with multi pattern", func() { So(ch.Scan(db, []byte("abctestdeftest"), 0, s, h.OnMatch, h.OnError, nil), ShouldBeNil) So(h.Events, ShouldResemble, []ch.MatchEvent{ - {0, 3, 7, 0, []*ch.Capture{{3, 7}}}, - {0, 10, 14, 0, []*ch.Capture{{10, 14}}}, + {0, 3, 7, 0, []*ch.Capture{{3, 7, []byte("test")}}}, + {0, 10, 14, 0, []*ch.Capture{{10, 14, []byte("test")}}}, }) }) From 5b8fdcdde6e62a52177d08d211bf59acf27acb00 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Thu, 30 Sep 2021 10:55:39 +0800 Subject: [PATCH 10/34] make go test -race happy --- .github/workflows/ci.yml | 10 +++++-- internal/ch/runtime.go | 14 +++++++-- internal/hs/runtime.go | 29 ++++++++++++++++--- internal/hs/stream.go | 62 +++++++++++++++++++++++++++++++++------- 4 files changed, 97 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07b1e04..cad885e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,11 +54,15 @@ jobs: - name: Test Hyperscan v4 API if: matrix.os == 'ubuntu-18.04' - run: go test -v -tags hyperscan_v4 ./internal/hs/... ./hyperscan/... + run: | + go test -v -tags hyperscan_v4 ./internal/hs/... ./hyperscan/... + go test -race -v -tags hyperscan_v4 ./internal/hs/... ./hyperscan/... - name: Test Hyperscan v5 API if: matrix.os != 'ubuntu-18.04' - run: go test -v ./internal/hs/... ./hyperscan/... + run: | + go test -v ./internal/hs/... ./hyperscan/... + go test -race -v ./internal/hs/... ./hyperscan/... build-and-test: strategy: @@ -157,6 +161,7 @@ jobs: CGO_LDFLAGS: -L${{ github.workspace }}/dist/lib run: | go test -v ${{ matrix.build_tags }} ./internal/hs/... ./hyperscan/... + go test -race -v ${{ matrix.build_tags }} ./internal/hs/... ./hyperscan/... - name: Test Chimera API if: startsWith(matrix.hyperscan, '5.') @@ -166,6 +171,7 @@ jobs: CGO_LDFLAGS: -L${{ github.workspace }}/dist/lib run: | go test -v ${{ matrix.build_tags }} ./internal/ch/... ./chimera/... + go test -trace -v ${{ matrix.build_tags }} ./internal/ch/... ./chimera/... golangci: name: lint diff --git a/internal/ch/runtime.go b/internal/ch/runtime.go index e3b4d92..1cdee5b 100644 --- a/internal/ch/runtime.go +++ b/internal/ch/runtime.go @@ -26,6 +26,16 @@ extern ch_callback_t errorEventCallback(ch_error_event_t error_type, unsigned int id, void *info, void *ctx); + +static inline +ch_error_t HS_CDECL _ch_scan(const ch_database_t *db, const char *data, + unsigned int length, unsigned int flags, + ch_scratch_t *scratch, + ch_match_event_handler onEvent, + ch_error_event_handler onError, + uintptr_t context) { + return ch_scan(db, data, length, flags, scratch, onEvent, onError, (void *)context); +} */ import "C" @@ -122,14 +132,14 @@ func Scan(db Database, data []byte, flags ScanFlag, scratch Scratch, hdr := (*reflect.SliceHeader)(unsafe.Pointer(&data)) // FIXME: Zero-copy access to go data - ret := C.ch_scan(db, + ret := C._ch_scan(db, (*C.char)(unsafe.Pointer(hdr.Data)), C.uint(hdr.Len), C.uint(flags), scratch, C.ch_match_event_handler(C.matchEventCallback), C.ch_error_event_handler(C.errorEventCallback), - unsafe.Pointer(h)) + C.uintptr_t(h)) // Ensure go data is alive before the C function returns runtime.KeepAlive(data) diff --git a/internal/hs/runtime.go b/internal/hs/runtime.go index dcc6078..c4d133a 100644 --- a/internal/hs/runtime.go +++ b/internal/hs/runtime.go @@ -17,6 +17,27 @@ extern int hsMatchEventCallback(unsigned int id, unsigned long long to, unsigned int flags, void *context); + +static inline +hs_error_t HS_CDECL _hs_scan(const hs_database_t *db, const char *data, + unsigned int length, unsigned int flags, + hs_scratch_t *scratch, match_event_handler onEvent, + uintptr_t context) +{ + return hs_scan(db, data, length, flags, scratch, onEvent, (void *)context); +} + +static inline +hs_error_t HS_CDECL _hs_scan_vector(const hs_database_t *db, + const char *const *data, + const unsigned int *length, + unsigned int count, unsigned int flags, + hs_scratch_t *scratch, + match_event_handler onEvent, + uintptr_t context) +{ + return hs_scan_vector(db, data, length, count, flags, scratch, onEvent, (void *)context); +} */ import "C" @@ -60,13 +81,13 @@ func Scan(db Database, data []byte, flags ScanFlag, scratch Scratch, onEvent Mat hdr := (*reflect.SliceHeader)(unsafe.Pointer(&data)) // FIXME: Zero-copy access to go data - ret := C.hs_scan(db, + ret := C._hs_scan(db, (*C.char)(unsafe.Pointer(hdr.Data)), C.uint(hdr.Len), C.uint(flags), scratch, C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) + C.uintptr_t(h)) // Ensure go data is alive before the C function returns runtime.KeepAlive(data) @@ -103,14 +124,14 @@ func ScanVector(db Database, data [][]byte, flags ScanFlag, scratch Scratch, onE cdataHdr := (*reflect.SliceHeader)(unsafe.Pointer(&cdata)) // FIXME: Zero-copy access to go data clengthHdr := (*reflect.SliceHeader)(unsafe.Pointer(&clength)) // FIXME: Zero-copy access to go data - ret := C.hs_scan_vector(db, + ret := C._hs_scan_vector(db, (**C.char)(unsafe.Pointer(cdataHdr.Data)), (*C.uint)(unsafe.Pointer(clengthHdr.Data)), C.uint(cdataHdr.Len), C.uint(flags), scratch, C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) + C.uintptr_t(h)) // Ensure go data is alive before the C function returns runtime.KeepAlive(data) diff --git a/internal/hs/stream.go b/internal/hs/stream.go index ddfe156..258b197 100644 --- a/internal/hs/stream.go +++ b/internal/hs/stream.go @@ -16,6 +16,48 @@ extern int hsMatchEventCallback(unsigned int id, unsigned long long to, unsigned int flags, void *context); + +static inline +hs_error_t HS_CDECL _hs_scan_stream(hs_stream_t *id, const char *data, + unsigned int length, unsigned int flags, + hs_scratch_t *scratch, + match_event_handler onEvent, + uintptr_t context) { + return hs_scan_stream(id, data, length, flags, scratch, onEvent, (void *)context); +} + +static inline +hs_error_t HS_CDECL _hs_close_stream(hs_stream_t *id, hs_scratch_t *scratch, + match_event_handler onEvent, + uintptr_t context) { + return hs_close_stream(id, scratch, onEvent, (void *)context); +} + +static inline +hs_error_t HS_CDECL _hs_reset_stream(hs_stream_t *id, unsigned int flags, + hs_scratch_t *scratch, + match_event_handler onEvent, + uintptr_t context) { + return hs_reset_stream(id, flags, scratch, onEvent, (void *)context); +} + +static inline +hs_error_t HS_CDECL _hs_reset_and_copy_stream(hs_stream_t *to_id, + const hs_stream_t *from_id, + hs_scratch_t *scratch, + match_event_handler onEvent, + uintptr_t context) { + return hs_reset_and_copy_stream(to_id, from_id, scratch, onEvent, (void *)context); +} + +static inline +hs_error_t HS_CDECL _hs_reset_and_expand_stream(hs_stream_t *to_stream, + const char *buf, size_t buf_size, + hs_scratch_t *scratch, + match_event_handler onEvent, + uintptr_t context) { + return hs_reset_and_expand_stream(to_stream, buf, buf_size, scratch, onEvent, (void *)context); +} */ import "C" @@ -41,13 +83,13 @@ func ScanStream(stream Stream, data []byte, flags ScanFlag, scratch Scratch, onE hdr := (*reflect.SliceHeader)(unsafe.Pointer(&data)) // FIXME: Zero-copy access to go data - ret := C.hs_scan_stream(stream, + ret := C._hs_scan_stream(stream, (*C.char)(unsafe.Pointer(hdr.Data)), C.uint(hdr.Len), C.uint(flags), scratch, C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) + C.uintptr_t(h)) // Ensure go data is alive before the C function returns runtime.KeepAlive(data) @@ -67,10 +109,10 @@ func CloseStream(stream Stream, scratch Scratch, onEvent MatchEventHandler, cont h := handle.New(MatchEventContext{onEvent, context}) defer h.Delete() - ret := C.hs_close_stream(stream, + ret := C._hs_close_stream(stream, scratch, C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) + C.uintptr_t(h)) if ret != C.HS_SUCCESS { return Error(ret) @@ -83,11 +125,11 @@ func ResetStream(stream Stream, flags ScanFlag, scratch Scratch, onEvent MatchEv h := handle.New(MatchEventContext{onEvent, context}) defer h.Delete() - ret := C.hs_reset_stream(stream, + ret := C._hs_reset_stream(stream, C.uint(flags), scratch, C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) + C.uintptr_t(h)) if ret != C.HS_SUCCESS { return Error(ret) @@ -110,11 +152,11 @@ func ResetAndCopyStream(to, from Stream, scratch Scratch, onEvent MatchEventHand h := handle.New(MatchEventContext{onEvent, context}) defer h.Delete() - ret := C.hs_reset_and_copy_stream(to, + ret := C._hs_reset_and_copy_stream(to, from, scratch, C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) + C.uintptr_t(h)) if ret != C.HS_SUCCESS { return Error(ret) @@ -157,12 +199,12 @@ func ResetAndExpandStream(stream Stream, buf []byte, scratch Scratch, onEvent Ma h := handle.New(MatchEventContext{onEvent, context}) defer h.Delete() - ret := C.hs_reset_and_expand_stream(stream, + ret := C._hs_reset_and_expand_stream(stream, (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)), scratch, C.match_event_handler(C.hsMatchEventCallback), - unsafe.Pointer(h)) + C.uintptr_t(h)) runtime.KeepAlive(buf) From 7a9f702415644c5c202e4f68549ce1dd90766f79 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Thu, 30 Sep 2021 10:58:49 +0800 Subject: [PATCH 11/34] upgrade dependencies --- go.mod | 9 ++-- go.sum | 8 ++++ .../smartystreets/assertions/go.mod | 2 +- .../x/sys/windows/syscall_windows.go | 5 +++ .../golang.org/x/sys/windows/types_windows.go | 14 ++++++ .../x/sys/windows/zsyscall_windows.go | 45 +++++++++++++++++++ vendor/modules.txt | 9 ++-- 7 files changed, 83 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 8c284dd..921737a 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.14 require ( github.com/google/gopacket v1.1.19 - github.com/gopherjs/gopherjs v0.0.0-20210918183006-daae65060a40 // indirect - github.com/smartystreets/assertions v1.2.0 // indirect + github.com/gopherjs/gopherjs v0.0.0-20210929192900-a930101f3d15 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/smartystreets/assertions v1.2.1 // indirect github.com/smartystreets/goconvey v1.6.4 - golang.org/x/net v0.0.0-20210917221730-978cfadd31cf // indirect - golang.org/x/sys v0.0.0-20210921065528-437939a70204 // indirect + golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 // indirect + golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect ) diff --git a/go.sum b/go.sum index 8894c20..415541f 100644 --- a/go.sum +++ b/go.sum @@ -140,6 +140,8 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20210918183006-daae65060a40 h1:AcvWELS0eUaNBj+uaWpwd18OSGuD9sf4KjDX4LMNbT4= github.com/gopherjs/gopherjs v0.0.0-20210918183006-daae65060a40/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI= +github.com/gopherjs/gopherjs v0.0.0-20210929192900-a930101f3d15 h1:+K5MaJ35/vUZwjZ+yOKLBCC0jytLRLHKIVTzi6fN/8w= +github.com/gopherjs/gopherjs v0.0.0-20210929192900-a930101f3d15/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -211,6 +213,8 @@ github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJV github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/assertions v1.2.1 h1:bKNHfEv7tSIjZ8JbKaFjzFINljxG4lzZvmHUnElzOIg= +github.com/smartystreets/assertions v1.2.1/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrxfxaaSlJazyFLYVFx8= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= @@ -326,6 +330,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210917221730-978cfadd31cf h1:R150MpwJIv1MpS0N/pc+NhTM8ajzvlmxlY5OYsrevXQ= golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 h1:Z04ewVs7JhXaYkmDhBERPi41gnltfQpMWDnTnQbaCqk= +golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -395,6 +401,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210921065528-437939a70204 h1:JJhkWtBuTQKyz2bd5WG9H8iUsJRU3En/KRfN8B2RnDs= golang.org/x/sys v0.0.0-20210921065528-437939a70204/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/vendor/github.com/smartystreets/assertions/go.mod b/vendor/github.com/smartystreets/assertions/go.mod index 3e0f123..7570e40 100644 --- a/vendor/github.com/smartystreets/assertions/go.mod +++ b/vendor/github.com/smartystreets/assertions/go.mod @@ -1,3 +1,3 @@ module github.com/smartystreets/assertions -go 1.13 +go 1.17 diff --git a/vendor/golang.org/x/sys/windows/syscall_windows.go b/vendor/golang.org/x/sys/windows/syscall_windows.go index 1215b2a..ff2d45d 100644 --- a/vendor/golang.org/x/sys/windows/syscall_windows.go +++ b/vendor/golang.org/x/sys/windows/syscall_windows.go @@ -398,6 +398,11 @@ func NewCallbackCDecl(fn interface{}) uintptr { // Process Status API (PSAPI) //sys EnumProcesses(processIds []uint32, bytesReturned *uint32) (err error) = psapi.EnumProcesses +//sys EnumProcessModules(process Handle, module *Handle, cb uint32, cbNeeded *uint32) (err error) = psapi.EnumProcessModules +//sys EnumProcessModulesEx(process Handle, module *Handle, cb uint32, cbNeeded *uint32, filterFlag uint32) (err error) = psapi.EnumProcessModulesEx +//sys GetModuleInformation(process Handle, module Handle, modinfo *ModuleInfo, cb uint32) (err error) = psapi.GetModuleInformation +//sys GetModuleFileNameEx(process Handle, module Handle, filename *uint16, size uint32) (err error) = psapi.GetModuleFileNameExW +//sys GetModuleBaseName(process Handle, module Handle, baseName *uint16, size uint32) (err error) = psapi.GetModuleBaseNameW // NT Native APIs //sys rtlNtStatusToDosErrorNoTeb(ntstatus NTStatus) (ret syscall.Errno) = ntdll.RtlNtStatusToDosErrorNoTeb diff --git a/vendor/golang.org/x/sys/windows/types_windows.go b/vendor/golang.org/x/sys/windows/types_windows.go index 17f0331..88e0ce5 100644 --- a/vendor/golang.org/x/sys/windows/types_windows.go +++ b/vendor/golang.org/x/sys/windows/types_windows.go @@ -242,6 +242,14 @@ const ( TH32CS_INHERIT = 0x80000000 ) +const ( + // flags for EnumProcessModulesEx + LIST_MODULES_32BIT = 0x01 + LIST_MODULES_64BIT = 0x02 + LIST_MODULES_ALL = 0x03 + LIST_MODULES_DEFAULT = 0x00 +) + const ( // filters for ReadDirectoryChangesW and FindFirstChangeNotificationW FILE_NOTIFY_CHANGE_FILE_NAME = 0x001 @@ -2773,3 +2781,9 @@ const ( // Flag for QueryFullProcessImageName. const PROCESS_NAME_NATIVE = 1 + +type ModuleInfo struct { + BaseOfDll uintptr + SizeOfImage uint32 + EntryPoint uintptr +} diff --git a/vendor/golang.org/x/sys/windows/zsyscall_windows.go b/vendor/golang.org/x/sys/windows/zsyscall_windows.go index 2083ec3..e282e24 100644 --- a/vendor/golang.org/x/sys/windows/zsyscall_windows.go +++ b/vendor/golang.org/x/sys/windows/zsyscall_windows.go @@ -377,7 +377,12 @@ var ( procCoTaskMemFree = modole32.NewProc("CoTaskMemFree") procCoUninitialize = modole32.NewProc("CoUninitialize") procStringFromGUID2 = modole32.NewProc("StringFromGUID2") + procEnumProcessModules = modpsapi.NewProc("EnumProcessModules") + procEnumProcessModulesEx = modpsapi.NewProc("EnumProcessModulesEx") procEnumProcesses = modpsapi.NewProc("EnumProcesses") + procGetModuleBaseNameW = modpsapi.NewProc("GetModuleBaseNameW") + procGetModuleFileNameExW = modpsapi.NewProc("GetModuleFileNameExW") + procGetModuleInformation = modpsapi.NewProc("GetModuleInformation") procSubscribeServiceChangeNotifications = modsechost.NewProc("SubscribeServiceChangeNotifications") procUnsubscribeServiceChangeNotifications = modsechost.NewProc("UnsubscribeServiceChangeNotifications") procGetUserNameExW = modsecur32.NewProc("GetUserNameExW") @@ -3225,6 +3230,22 @@ func stringFromGUID2(rguid *GUID, lpsz *uint16, cchMax int32) (chars int32) { return } +func EnumProcessModules(process Handle, module *Handle, cb uint32, cbNeeded *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEnumProcessModules.Addr(), 4, uintptr(process), uintptr(unsafe.Pointer(module)), uintptr(cb), uintptr(unsafe.Pointer(cbNeeded)), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func EnumProcessModulesEx(process Handle, module *Handle, cb uint32, cbNeeded *uint32, filterFlag uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEnumProcessModulesEx.Addr(), 5, uintptr(process), uintptr(unsafe.Pointer(module)), uintptr(cb), uintptr(unsafe.Pointer(cbNeeded)), uintptr(filterFlag), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func EnumProcesses(processIds []uint32, bytesReturned *uint32) (err error) { var _p0 *uint32 if len(processIds) > 0 { @@ -3237,6 +3258,30 @@ func EnumProcesses(processIds []uint32, bytesReturned *uint32) (err error) { return } +func GetModuleBaseName(process Handle, module Handle, baseName *uint16, size uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procGetModuleBaseNameW.Addr(), 4, uintptr(process), uintptr(module), uintptr(unsafe.Pointer(baseName)), uintptr(size), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func GetModuleFileNameEx(process Handle, module Handle, filename *uint16, size uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procGetModuleFileNameExW.Addr(), 4, uintptr(process), uintptr(module), uintptr(unsafe.Pointer(filename)), uintptr(size), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func GetModuleInformation(process Handle, module Handle, modinfo *ModuleInfo, cb uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procGetModuleInformation.Addr(), 4, uintptr(process), uintptr(module), uintptr(unsafe.Pointer(modinfo)), uintptr(cb), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func SubscribeServiceChangeNotifications(service Handle, eventType uint32, callback uintptr, callbackCtx uintptr, subscription *uintptr) (ret error) { ret = procSubscribeServiceChangeNotifications.Find() if ret != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index 5d3b9e4..af9b74a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -3,12 +3,13 @@ github.com/google/gopacket github.com/google/gopacket/layers github.com/google/gopacket/pcap -# github.com/gopherjs/gopherjs v0.0.0-20210918183006-daae65060a40 +# github.com/gopherjs/gopherjs v0.0.0-20210929192900-a930101f3d15 ## explicit github.com/gopherjs/gopherjs/js # github.com/jtolds/gls v4.20.0+incompatible +## explicit github.com/jtolds/gls -# github.com/smartystreets/assertions v1.2.0 +# github.com/smartystreets/assertions v1.2.1 ## explicit github.com/smartystreets/assertions github.com/smartystreets/assertions/internal/go-diff/diffmatchpatch @@ -19,9 +20,9 @@ github.com/smartystreets/assertions/internal/oglematchers github.com/smartystreets/goconvey/convey github.com/smartystreets/goconvey/convey/gotest github.com/smartystreets/goconvey/convey/reporting -# golang.org/x/net v0.0.0-20210917221730-978cfadd31cf +# golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 ## explicit -# golang.org/x/sys v0.0.0-20210921065528-437939a70204 +# golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 ## explicit golang.org/x/sys/internal/unsafeheader golang.org/x/sys/windows From 445256f90ab0ed08ced05067fd6867f4db5d85f5 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Thu, 30 Sep 2021 11:08:06 +0800 Subject: [PATCH 12/34] run and upload coverage to Codecov --- .github/workflows/ci.yml | 20 +++++++++++--------- .gitignore | 2 ++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cad885e..8195b1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,10 @@ jobs: build_tags: -tags hyperscan_v4 name: Go ${{ matrix.go }} tests @ ${{ matrix.os }} for hyperscan ${{ matrix.hyperscan }} runs-on: ${{ matrix.os }} + env: + PKG_CONFIG_PATH: ${{ github.workspace }}/dist/lib/pkgconfig + CGO_CFLAGS: -I${{ github.workspace }}/dist/include/hs + CGO_LDFLAGS: -L${{ github.workspace }}/dist/lib steps: - name: Install Linux dependencies if: startsWith(matrix.os, 'ubuntu-') @@ -155,23 +159,21 @@ jobs: ${{ runner.os }}-go- - name: Test Hyperscan API - env: - PKG_CONFIG_PATH: ${{ github.workspace }}/dist/lib/pkgconfig - CGO_CFLAGS: -I${{ github.workspace }}/dist/include/hs - CGO_LDFLAGS: -L${{ github.workspace }}/dist/lib run: | go test -v ${{ matrix.build_tags }} ./internal/hs/... ./hyperscan/... go test -race -v ${{ matrix.build_tags }} ./internal/hs/... ./hyperscan/... - name: Test Chimera API if: startsWith(matrix.hyperscan, '5.') - env: - PKG_CONFIG_PATH: ${{ github.workspace }}/dist/lib/pkgconfig - CGO_CFLAGS: -I${{ github.workspace }}/dist/include/hs - CGO_LDFLAGS: -L${{ github.workspace }}/dist/lib run: | go test -v ${{ matrix.build_tags }} ./internal/ch/... ./chimera/... - go test -trace -v ${{ matrix.build_tags }} ./internal/ch/... ./chimera/... + go test -race -v ${{ matrix.build_tags }} ./internal/ch/... ./chimera/... + + - name: Run and upload coverage to Codecov + if: startsWith(matrix.hyperscan, '5.4.') + run: | + go test -race -coverprofile=coverage.out -covermode=atomic ./... + bash <(curl -s https://codecov.io/bash) golangci: name: lint diff --git a/.gitignore b/.gitignore index 0c9e329..5aec6a4 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ _testmain.go *.exe *.test *.prof + +*.out From 46b29b2dd0ce108b1eb0679f54d8aa24115774a1 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Thu, 30 Sep 2021 11:13:22 +0800 Subject: [PATCH 13/34] include --- internal/ch/runtime.go | 2 ++ internal/hs/runtime.go | 2 ++ internal/hs/stream.go | 2 ++ 3 files changed, 6 insertions(+) diff --git a/internal/ch/runtime.go b/internal/ch/runtime.go index 1cdee5b..5410d9a 100644 --- a/internal/ch/runtime.go +++ b/internal/ch/runtime.go @@ -10,6 +10,8 @@ import ( ) /* +#include + #include typedef const ch_capture_t capture_t; diff --git a/internal/hs/runtime.go b/internal/hs/runtime.go index c4d133a..f9781c0 100644 --- a/internal/hs/runtime.go +++ b/internal/hs/runtime.go @@ -10,6 +10,8 @@ import ( ) /* +#include + #include extern int hsMatchEventCallback(unsigned int id, diff --git a/internal/hs/stream.go b/internal/hs/stream.go index 258b197..26f4101 100644 --- a/internal/hs/stream.go +++ b/internal/hs/stream.go @@ -9,6 +9,8 @@ import ( ) /* +#include + #include extern int hsMatchEventCallback(unsigned int id, From c4079fab63050f6cb87531695184c011e0960d9b Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Thu, 30 Sep 2021 11:28:03 +0800 Subject: [PATCH 14/34] use CODECOV_TOKEN from secrets --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8195b1f..3a080fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,8 +106,6 @@ jobs: brew update brew install pkg-config libpcap ragel cmake boost ninja lzlib wget tree - - uses: actions/checkout@v2 - - name: Cache Hyperscan library id: cache-hyperscan uses: actions/cache@v2 @@ -158,6 +156,8 @@ jobs: restore-keys: | ${{ runner.os }}-go- + - uses: actions/checkout@v2 + - name: Test Hyperscan API run: | go test -v ${{ matrix.build_tags }} ./internal/hs/... ./hyperscan/... @@ -171,6 +171,8 @@ jobs: - name: Run and upload coverage to Codecov if: startsWith(matrix.hyperscan, '5.4.') + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} run: | go test -race -coverprofile=coverage.out -covermode=atomic ./... bash <(curl -s https://codecov.io/bash) From ab804197c89fd10c8a2f403d50c2db6f6c756e92 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Thu, 30 Sep 2021 11:39:49 +0800 Subject: [PATCH 15/34] run coverage base on matrix setting --- .github/workflows/ci.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a080fe..bf4e315 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,21 +71,22 @@ jobs: - os: macos-latest hyperscan: 5.4.0 pcre: 8.45 - build_tags: -tags hyperscan_v54 + build_flags: -tags hyperscan_v54 - os: macos-latest hyperscan: 5.2.1 pcre: 8.45 - os: ubuntu-20.04 hyperscan: 5.4.0 pcre: 8.45 - build_tags: -tags hyperscan_v54 + build_flags: -tags hyperscan_v54 + coverage: true - os: ubuntu-20.04 hyperscan: 5.2.1 pcre: 8.45 - os: ubuntu-18.04 hyperscan: 4.7.0 pcre: 8.41 - build_tags: -tags hyperscan_v4 + build_flags: -tags hyperscan_v4 name: Go ${{ matrix.go }} tests @ ${{ matrix.os }} for hyperscan ${{ matrix.hyperscan }} runs-on: ${{ matrix.os }} env: @@ -106,6 +107,8 @@ jobs: brew update brew install pkg-config libpcap ragel cmake boost ninja lzlib wget tree + - uses: actions/checkout@v2 + - name: Cache Hyperscan library id: cache-hyperscan uses: actions/cache@v2 @@ -156,21 +159,19 @@ jobs: restore-keys: | ${{ runner.os }}-go- - - uses: actions/checkout@v2 - - name: Test Hyperscan API run: | - go test -v ${{ matrix.build_tags }} ./internal/hs/... ./hyperscan/... - go test -race -v ${{ matrix.build_tags }} ./internal/hs/... ./hyperscan/... + go test -v ${{ matrix.build_flags }} ./internal/hs/... ./hyperscan/... + go test -race -v ${{ matrix.build_flags }} ./internal/hs/... ./hyperscan/... - name: Test Chimera API - if: startsWith(matrix.hyperscan, '5.') + if: matrix.coverage run: | - go test -v ${{ matrix.build_tags }} ./internal/ch/... ./chimera/... - go test -race -v ${{ matrix.build_tags }} ./internal/ch/... ./chimera/... + go test -v ${{ matrix.build_flags }} ./internal/ch/... ./chimera/... + go test -race -v ${{ matrix.build_flags }} ./internal/ch/... ./chimera/... - name: Run and upload coverage to Codecov - if: startsWith(matrix.hyperscan, '5.4.') + if: matrix.coverage env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} run: | From ce63ea8183ac9baa4dad6e2597a2d03bba167b67 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Thu, 30 Sep 2021 12:40:28 +0800 Subject: [PATCH 16/34] add codecov badge --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a9dedbd..09ba13e 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ -# gohs [![Continuous integration](https://github.com/flier/gohs/actions/workflows/ci.yml/badge.svg?)](https://github.com/flier/gohs/actions/workflows/ci.yml) [![Go Report Card](https://goreportcard.com/badge/github.com/flier/gohs?)](https://goreportcard.com/report/github.com/flier/gohs) [![Go Reference](https://pkg.go.dev/badge/github.com/flier/gohs/hyperscan.svg)](https://pkg.go.dev/github.com/flier/gohs/hyperscan) [![Apache](https://img.shields.io/badge/license-Apache-blue.svg)](https://github.com/flier/gohs/blob/master/LICENSE-APACHE) [![MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/flier/gohs/blob/master/LICENSE-MIT) +# gohs [![Continuous integration](https://github.com/flier/gohs/actions/workflows/ci.yml/badge.svg?)](https://github.com/flier/gohs/actions/workflows/ci.yml) [![Go Report Card](https://goreportcard.com/badge/github.com/flier/gohs?)](https://goreportcard.com/report/github.com/flier/gohs) [![codecov](https://codecov.io/gh/flier/gohs/branch/master/graph/badge.svg?token=F5CLCxpJGM)](https://codecov.io/gh/flier/gohs) [![Apache](https://img.shields.io/badge/license-Apache-blue.svg)](https://github.com/flier/gohs/blob/master/LICENSE-APACHE) [![MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/flier/gohs/blob/master/LICENSE-MIT) Golang binding for Intel's HyperScan regex matching library: [hyperscan.io](https://www.hyperscan.io/) -## Build +## Hyperscan [![Go Reference](https://pkg.go.dev/badge/github.com/flier/gohs/hyperscan.svg)](https://pkg.go.dev/github.com/flier/gohs/hyperscan) -**Note:** `gohs` will use Hyperscan v5 API by default, you can also build for Hyperscan v4 with `hyperscan_v4` tag. +Hyperscan is a software regular expression matching engine designed with high performance and flexibility in mind. It is implemented as a library that exposes a straightforward C API. + +### Build + +`gohs` will use Hyperscan v5 API by default, you can also build for Hyperscan v4 with `hyperscan_v4` tag. ```bash go get -u -tags hyperscan_v4 github.com/flier/gohs/hyperscan @@ -14,6 +18,8 @@ go get -u -tags hyperscan_v4 github.com/flier/gohs/hyperscan Chimera is a software regular expression matching engine that is a hybrid of Hyperscan and PCRE. The design goals of Chimera are to fully support PCRE syntax as well as to take advantage of the high performance nature of Hyperscan. +### Build + It is recommended to compile and link Chimera using static libraries. ```bash From 5a93585f99d7ac91833dd3a33275339ff8222e08 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Wed, 24 Nov 2021 13:13:07 +0800 Subject: [PATCH 17/34] replace Expression with string for back compatibility --- chimera/compile.go | 4 ++-- chimera/pattern.go | 9 +++------ examples/simplegrep/main.go | 2 +- hyperscan/compile.go | 4 ++-- hyperscan/literal.go | 17 +++++++---------- hyperscan/pattern.go | 32 ++++++++++++-------------------- internal/ch/compile.go | 17 ++++++++++------- 7 files changed, 37 insertions(+), 48 deletions(-) diff --git a/chimera/compile.go b/chimera/compile.go index 8ce4a8b..c66bb06 100644 --- a/chimera/compile.go +++ b/chimera/compile.go @@ -119,7 +119,7 @@ type DatabaseBuilder struct { } // AddExpressions add more expressions to the database. -func (b *DatabaseBuilder) AddExpressions(exprs ...Expression) *DatabaseBuilder { +func (b *DatabaseBuilder) AddExpressions(exprs ...string) *DatabaseBuilder { for _, expr := range exprs { b.Patterns = append(b.Patterns, &Pattern{Expression: expr, Id: len(b.Patterns) + 1}) } @@ -128,7 +128,7 @@ func (b *DatabaseBuilder) AddExpressions(exprs ...Expression) *DatabaseBuilder { } // AddExpressionWithFlags add more expressions with flags to the database. -func (b *DatabaseBuilder) AddExpressionWithFlags(expr Expression, flags CompileFlag) *DatabaseBuilder { +func (b *DatabaseBuilder) AddExpressionWithFlags(expr string, flags CompileFlag) *DatabaseBuilder { b.Patterns = append(b.Patterns, &Pattern{Expression: expr, Flags: flags, Id: len(b.Patterns) + 1}) return b diff --git a/chimera/pattern.go b/chimera/pattern.go index 0d4d916..9720e7e 100644 --- a/chimera/pattern.go +++ b/chimera/pattern.go @@ -10,15 +10,12 @@ import ( "github.com/flier/gohs/internal/ch" ) -// Expression of pattern. -type Expression = ch.Expression - // Pattern is a matching pattern. type Pattern ch.Pattern // NewPattern returns a new pattern base on expression and compile flags. func NewPattern(expr string, flags CompileFlag) *Pattern { - return &Pattern{Expression: Expression(expr), Flags: flags} + return &Pattern{Expression: expr, Flags: flags} } /* @@ -48,7 +45,7 @@ func ParsePattern(s string) (*Pattern, error) { } if n := strings.LastIndex(s, "/"); n > 1 && strings.HasPrefix(s, "/") { - p.Expression = Expression(s[1:n]) + p.Expression = s[1:n] s = s[n+1:] flags, err := ParseCompileFlag(s) @@ -58,7 +55,7 @@ func ParsePattern(s string) (*Pattern, error) { p.Flags = flags } else { - p.Expression = Expression(s) + p.Expression = s } return &p, nil diff --git a/examples/simplegrep/main.go b/examples/simplegrep/main.go index cd3afc3..0319dbb 100644 --- a/examples/simplegrep/main.go +++ b/examples/simplegrep/main.go @@ -104,7 +104,7 @@ func main() { } } - pattern := hyperscan.NewPattern(hyperscan.Expression(flag.Arg(0)), hyperscan.DotAll|hyperscan.SomLeftMost) + pattern := hyperscan.NewPattern(flag.Arg(0), hyperscan.DotAll|hyperscan.SomLeftMost) /* First, we attempt to compile the pattern provided on the command line. * We assume 'DOTALL' semantics, meaning that the '.' meta-character will diff --git a/hyperscan/compile.go b/hyperscan/compile.go index 2d7e7d5..31929cb 100644 --- a/hyperscan/compile.go +++ b/hyperscan/compile.go @@ -143,7 +143,7 @@ type DatabaseBuilder struct { } // AddExpressions add more expressions to the database. -func (b *DatabaseBuilder) AddExpressions(exprs ...Expression) *DatabaseBuilder { +func (b *DatabaseBuilder) AddExpressions(exprs ...string) *DatabaseBuilder { for _, expr := range exprs { b.Patterns = append(b.Patterns, &Pattern{Expression: expr, Id: len(b.Patterns) + 1}) } @@ -152,7 +152,7 @@ func (b *DatabaseBuilder) AddExpressions(exprs ...Expression) *DatabaseBuilder { } // AddExpressionWithFlags add more expressions with flags to the database. -func (b *DatabaseBuilder) AddExpressionWithFlags(expr Expression, flags CompileFlag) *DatabaseBuilder { +func (b *DatabaseBuilder) AddExpressionWithFlags(expr string, flags CompileFlag) *DatabaseBuilder { b.Patterns = append(b.Patterns, &Pattern{Expression: expr, Flags: flags, Id: len(b.Patterns) + 1}) return b diff --git a/hyperscan/literal.go b/hyperscan/literal.go index cfc6e44..b5b9e24 100644 --- a/hyperscan/literal.go +++ b/hyperscan/literal.go @@ -19,12 +19,9 @@ type Literals []*Literal // No syntax association happens between any adjacent characters. // nolint: golint,revive,stylecheck type Literal struct { - // The expression to parse. - Expression - // Flags which modify the behaviour of the expression. - Flags CompileFlag - // The ID number to be associated with the corresponding pattern - Id int + Expression string // The expression to parse. + Flags CompileFlag // Flags which modify the behaviour of the expression. + Id int // The ID number to be associated with the corresponding pattern *ExprInfo } @@ -34,7 +31,7 @@ func NewLiteral(expr string, flags ...CompileFlag) *Literal { for _, f := range flags { v |= f } - return &Literal{Expression: Expression(expr), Flags: v} + return &Literal{Expression: expr, Flags: v} } // IsValid validate the literal contains a pure literal. @@ -95,7 +92,7 @@ func ParseLiteral(s string) (*Literal, error) { } if n := strings.LastIndex(s, "/"); n > 1 && strings.HasPrefix(s, "/") { - lit.Expression = Expression(s[1:n]) + lit.Expression = s[1:n] s = s[n+1:] flags, err := ParseCompileFlag(s) @@ -104,10 +101,10 @@ func ParseLiteral(s string) (*Literal, error) { } lit.Flags = flags } else { - lit.Expression = Expression(s) + lit.Expression = s } - info, err := hs.ExpressionInfo(string(lit.Expression), lit.Flags) + info, err := hs.ExpressionInfo(lit.Expression, lit.Flags) if err != nil { return nil, fmt.Errorf("invalid pattern `%s`, %w", lit.Expression, err) } diff --git a/hyperscan/pattern.go b/hyperscan/pattern.go index 4112e68..a190e6e 100644 --- a/hyperscan/pattern.go +++ b/hyperscan/pattern.go @@ -176,26 +176,18 @@ func ParseExprExt(s string) (ext *ExprExt, err error) { return // nolint: nakedret } -// Expression of pattern. -type Expression string - -func (e Expression) String() string { return string(e) } - // Pattern is a matching pattern. // nolint: golint,revive,stylecheck type Pattern struct { - // The expression to parse. - Expression - // Flags which modify the behaviour of the expression. - Flags CompileFlag - // The ID number to be associated with the corresponding pattern - Id int - info *ExprInfo - ext *ExprExt + Expression string // The expression to parse. + Flags CompileFlag // Flags which modify the behaviour of the expression. + Id int // The ID number to be associated with the corresponding pattern + info *ExprInfo + ext *ExprExt } // NewPattern returns a new pattern base on expression and compile flags. -func NewPattern(expr Expression, flags CompileFlag, exts ...Ext) *Pattern { +func NewPattern(expr string, flags CompileFlag, exts ...Ext) *Pattern { return &Pattern{ Expression: expr, Flags: flags, @@ -205,7 +197,7 @@ func NewPattern(expr Expression, flags CompileFlag, exts ...Ext) *Pattern { func (p *Pattern) Pattern() *hs.Pattern { return &hs.Pattern{ - Expr: string(p.Expression), + Expr: p.Expression, Flags: p.Flags, ID: p.Id, Ext: (*hs.ExprExt)(p.ext), @@ -226,7 +218,7 @@ func (p *Pattern) IsValid() bool { // Info provides information about a regular expression. func (p *Pattern) Info() (*ExprInfo, error) { if p.info == nil { - info, err := hs.ExpressionInfo(string(p.Expression), p.Flags) + info, err := hs.ExpressionInfo(p.Expression, p.Flags) if err != nil { return nil, err // nolint: wrapcheck } @@ -251,7 +243,7 @@ func (p *Pattern) WithExt(exts ...Ext) *Pattern { // Ext provides additional parameters related to an expression. func (p *Pattern) Ext() (*ExprExt, error) { if p.ext == nil { - ext, info, err := hs.ExpressionExt(string(p.Expression), p.Flags) + ext, info, err := hs.ExpressionExt(p.Expression, p.Flags) if err != nil { return nil, err // nolint: wrapcheck } @@ -306,7 +298,7 @@ func ParsePattern(s string) (*Pattern, error) { } if n := strings.LastIndex(s, "/"); n > 1 && strings.HasPrefix(s, "/") { - p.Expression = Expression(s[1:n]) + p.Expression = s[1:n] s = s[n+1:] if n = strings.Index(s, "{"); n > 0 && strings.HasSuffix(s, "}") { @@ -326,10 +318,10 @@ func ParsePattern(s string) (*Pattern, error) { p.Flags = flags } else { - p.Expression = Expression(s) + p.Expression = s } - info, err := hs.ExpressionInfo(string(p.Expression), p.Flags) + info, err := hs.ExpressionInfo(p.Expression, p.Flags) if err != nil { return nil, fmt.Errorf("pattern `%s`, %w", p.Expression, err) } diff --git a/internal/ch/compile.go b/internal/ch/compile.go index b4ba848..c07fd3d 100644 --- a/internal/ch/compile.go +++ b/internal/ch/compile.go @@ -13,19 +13,22 @@ import ( // #include import "C" -// Expression of pattern. -type Expression string - -func (e Expression) String() string { return string(e) } - // Pattern is a matching pattern. // nolint: golint,revive,stylecheck type Pattern struct { - Expression // The expression to parse. + Expression string // The expression to parse. Flags CompileFlag // Flags which modify the behaviour of the expression. Id int // The ID number to be associated with the corresponding pattern } +// NewPattern returns a new pattern base on expression and compile flags. +func NewPattern(expr string, flags CompileFlag) *Pattern { + return &Pattern{ + Expression: expr, + Flags: flags, + } +} + // A type containing error details that is returned by the compile calls on failure. // // The caller may inspect the values returned in this type to determine the cause of failure. @@ -150,7 +153,7 @@ func CompileExtMulti(p Patterns, mode CompileMode, info *hs.PlatformInfo, ids := (*[1 << 30]C.uint)(unsafe.Pointer(cids))[:len(patterns):len(patterns)] for i, pattern := range patterns { - exprs[i] = C.CString(string(pattern.Expression)) + exprs[i] = C.CString(pattern.Expression) flags[i] = C.uint(pattern.Flags) ids[i] = C.uint(pattern.Id) } From f1235087bdd628f43c01f3bc810a8f960b71ad08 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Wed, 24 Nov 2021 13:16:53 +0800 Subject: [PATCH 18/34] add benchmark for chimera --- bench/go/scan_test.go | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/bench/go/scan_test.go b/bench/go/scan_test.go index fd2ff23..a9cb1f1 100644 --- a/bench/go/scan_test.go +++ b/bench/go/scan_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/flier/gohs/chimera" "github.com/flier/gohs/hyperscan" ) @@ -73,7 +74,7 @@ func BenchmarkHyperscanBlockScan(b *testing.B) { } m := func(id uint, from, to uint64, flags uint, context interface{}) error { - return hyperscan.ErrUnexpected + return hyperscan.ErrScanTerminated } for _, size := range benchSizes { @@ -111,7 +112,7 @@ func BenchmarkHyperscanStreamScan(b *testing.B) { } m := func(id uint, from, to uint64, flags uint, context interface{}) error { - return hyperscan.ErrUnexpected + return hyperscan.ErrScanTerminated } for _, size := range benchSizes { @@ -142,6 +143,43 @@ func BenchmarkHyperscanStreamScan(b *testing.B) { } } +func BenchmarkChimeraBlockScan(b *testing.B) { + isRaceBuilder := strings.HasSuffix(testenv(), "-race") + + for _, data := range benchData { + p := chimera.NewPattern(data.re, chimera.MultiLine) + db, err := chimera.NewBlockDatabase(p) + if err != nil { + b.Fatalf("compile pattern %s: `%s`, %s", data.name, data.re, err) + } + + s, err := chimera.NewScratch(db) + if err != nil { + b.Fatalf("create scratch, %s", err) + } + + m := chimera.HandlerFunc(func(id uint, from, to uint64, flags uint, + captured []*chimera.Capture, context interface{}) chimera.Callback { + return chimera.Terminate + }) + + for _, size := range benchSizes { + if (isRaceBuilder || testing.Short()) && size.n > 1<<10 { + continue + } + t := makeText(size.n) + b.Run(data.name+"/"+size.name, func(b *testing.B) { + b.SetBytes(int64(len(t))) + for i := 0; i < b.N; i++ { + if err = db.Scan(t, s, m, nil); err != nil { + b.Fatalf("match, %s", err) + } + } + }) + } + } +} + func BenchmarkRegexpMatch(b *testing.B) { isRaceBuilder := strings.HasSuffix(testenv(), "-race") From fce3c0e6286ee502f1908806ed69102e0a34a27c Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Wed, 24 Nov 2021 13:35:06 +0800 Subject: [PATCH 19/34] ignore scan flags in MatchRecorder.Handle --- internal/hs/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/hs/runtime.go b/internal/hs/runtime.go index 8603aaf..f560ec0 100644 --- a/internal/hs/runtime.go +++ b/internal/hs/runtime.go @@ -144,7 +144,7 @@ func (h *MatchRecorder) Handle(id uint, from, to uint64, flags uint, context int if len(h.Events) > 0 { tail := &h.Events[len(h.Events)-1] - if tail.ID == id && tail.From == from && tail.ScanFlag == ScanFlag(flags) && tail.To < to { + if tail.ID == id && tail.From == from && tail.To < to { tail.To = to return h.Err From f7edc136866ee79aabe0cee522c8f3760bfa149f Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Wed, 24 Nov 2021 15:55:04 +0800 Subject: [PATCH 20/34] fix lint warning --- .gitignore | 1 + .golangci.yml | 2 ++ bench/go/scan_test.go | 9 +++++---- chimera/compile.go | 4 ++-- chimera/example_compile_test.go | 2 +- chimera/pattern.go | 6 +++--- chimera/pattern_test.go | 4 ++-- hyperscan/literal.go | 6 +++--- hyperscan/literal_test.go | 2 +- hyperscan/pattern_test.go | 2 +- internal/ch/allocator.go | 2 +- internal/ch/common_test.go | 6 ++++-- internal/ch/compile.go | 5 ++--- internal/ch/runtime.go | 6 +++--- 14 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 34aab97..68ee1a9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ _obj _test .idea +.vscode build # Architecture specific extensions/prefixes diff --git a/.golangci.yml b/.golangci.yml index 2ccd064..94f1b6c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -75,6 +75,8 @@ linters: - gochecknoinits - gocritic - godox + - ireturn - nlreturn - paralleltest + - varnamelen - wsl diff --git a/bench/go/scan_test.go b/bench/go/scan_test.go index a9cb1f1..463c79b 100644 --- a/bench/go/scan_test.go +++ b/bench/go/scan_test.go @@ -1,4 +1,4 @@ -package tests +package scan_test import ( "os" @@ -96,6 +96,7 @@ func BenchmarkHyperscanBlockScan(b *testing.B) { const PageSize = 4096 +// nolint: gocognit func BenchmarkHyperscanStreamScan(b *testing.B) { isRaceBuilder := strings.HasSuffix(testenv(), "-race") @@ -127,8 +128,8 @@ func BenchmarkHyperscanStreamScan(b *testing.B) { if err != nil { b.Fatalf("open stream, %s", err) } - for i := 0; i < size.n; i += PageSize { - n := size.n - i + for i := 0; i < len(t); i += PageSize { + n := len(t) - i if n > PageSize { n = PageSize } @@ -191,7 +192,7 @@ func BenchmarkRegexpMatch(b *testing.B) { } t := makeText(size.n) b.Run(data.name+"/"+size.name, func(b *testing.B) { - b.SetBytes(int64(size.n)) + b.SetBytes(int64(len(t))) for i := 0; i < b.N; i++ { if r.Match(t) { b.Fatal("match!") diff --git a/chimera/compile.go b/chimera/compile.go index c66bb06..4c37e1e 100644 --- a/chimera/compile.go +++ b/chimera/compile.go @@ -121,7 +121,7 @@ type DatabaseBuilder struct { // AddExpressions add more expressions to the database. func (b *DatabaseBuilder) AddExpressions(exprs ...string) *DatabaseBuilder { for _, expr := range exprs { - b.Patterns = append(b.Patterns, &Pattern{Expression: expr, Id: len(b.Patterns) + 1}) + b.Patterns = append(b.Patterns, &Pattern{Expression: expr, ID: len(b.Patterns) + 1}) } return b @@ -129,7 +129,7 @@ func (b *DatabaseBuilder) AddExpressions(exprs ...string) *DatabaseBuilder { // AddExpressionWithFlags add more expressions with flags to the database. func (b *DatabaseBuilder) AddExpressionWithFlags(expr string, flags CompileFlag) *DatabaseBuilder { - b.Patterns = append(b.Patterns, &Pattern{Expression: expr, Flags: flags, Id: len(b.Patterns) + 1}) + b.Patterns = append(b.Patterns, &Pattern{Expression: expr, Flags: flags, ID: len(b.Patterns) + 1}) return b } diff --git a/chimera/example_compile_test.go b/chimera/example_compile_test.go index a0d039e..8cf20dd 100644 --- a/chimera/example_compile_test.go +++ b/chimera/example_compile_test.go @@ -29,7 +29,7 @@ func ExampleParsePattern() { p, err := chimera.ParsePattern("3:/foobar/i8") fmt.Println(err) - fmt.Println(p.Id) + fmt.Println(p.ID) fmt.Println(p.Expression) fmt.Println(p.Flags) diff --git a/chimera/pattern.go b/chimera/pattern.go index 9720e7e..a9d6f38 100644 --- a/chimera/pattern.go +++ b/chimera/pattern.go @@ -40,7 +40,7 @@ func ParsePattern(s string) (*Pattern, error) { return nil, fmt.Errorf("pattern id `%s`, %w", s[:i], ErrInvalid) } - p.Id = id + p.ID = id s = s[i+1:] } @@ -64,8 +64,8 @@ func ParsePattern(s string) (*Pattern, error) { func (p *Pattern) String() string { var b strings.Builder - if p.Id > 0 { - fmt.Fprintf(&b, "%d:", p.Id) + if p.ID > 0 { + fmt.Fprintf(&b, "%d:", p.ID) } fmt.Fprintf(&b, "/%s/%s", p.Expression, p.Flags) diff --git a/chimera/pattern_test.go b/chimera/pattern_test.go index 2e87fc0..6c5856d 100644 --- a/chimera/pattern_test.go +++ b/chimera/pattern_test.go @@ -19,7 +19,7 @@ func TestPattern(t *testing.T) { So(p.Expression, ShouldEqual, "test") So(p.Flags, ShouldEqual, chimera.Caseless|chimera.MultiLine) - So(string(p.Expression), ShouldEqual, "test") + So(p.Expression, ShouldEqual, "test") So(p.String(), ShouldEqual, `/test/im`) Convey("When pattern contains forward slash", func() { @@ -38,7 +38,7 @@ func TestPattern(t *testing.T) { p, err := chimera.ParsePattern("3:/foobar/i8") So(err, ShouldBeNil) - So(p.Id, ShouldEqual, 3) + So(p.ID, ShouldEqual, 3) So(p.Expression, ShouldEqual, "foobar") So(p.Flags, ShouldEqual, chimera.Caseless|chimera.Utf8Mode) }) diff --git a/hyperscan/literal.go b/hyperscan/literal.go index b5b9e24..7e1eb43 100644 --- a/hyperscan/literal.go +++ b/hyperscan/literal.go @@ -44,7 +44,7 @@ func (lit *Literal) IsValid() bool { // Provides information about a regular expression. func (lit *Literal) Info() (*ExprInfo, error) { if lit.ExprInfo == nil { - info, err := hs.ExpressionInfo(string(lit.Expression), lit.Flags) + info, err := hs.ExpressionInfo(lit.Expression, lit.Flags) if err != nil { return nil, err // nolint: wrapcheck } @@ -115,7 +115,7 @@ func ParseLiteral(s string) (*Literal, error) { func (lit *Literal) Literal() *hs.Literal { return &hs.Literal{ - Expr: string(lit.Expression), + Expr: lit.Expression, Flags: lit.Flags, ID: lit.Id, ExprInfo: lit.ExprInfo, @@ -143,7 +143,7 @@ func (lit *Literal) ForPlatform(mode ModeFlag, platform Platform) (Database, err p, _ := platform.(*hs.PlatformInfo) - db, err := hs.CompileLit(string(lit.Expression), lit.Flags, mode, p) + db, err := hs.CompileLit(lit.Expression, lit.Flags, mode, p) if err != nil { return nil, err // nolint: wrapcheck } diff --git a/hyperscan/literal_test.go b/hyperscan/literal_test.go index e675a43..de93b1d 100644 --- a/hyperscan/literal_test.go +++ b/hyperscan/literal_test.go @@ -22,7 +22,7 @@ func TestLiteral(t *testing.T) { So(p.Expression, ShouldEqual, "test") So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) - So(string(p.Expression), ShouldEqual, "test") + So(p.Expression, ShouldEqual, "test") So(p.String(), ShouldEqual, `/test/im`) Convey("When literal contains regular grammar", func() { diff --git a/hyperscan/pattern_test.go b/hyperscan/pattern_test.go index 15f2532..6a9202c 100644 --- a/hyperscan/pattern_test.go +++ b/hyperscan/pattern_test.go @@ -19,7 +19,7 @@ func TestPattern(t *testing.T) { So(p.Expression, ShouldEqual, "test") So(p.Flags, ShouldEqual, hyperscan.Caseless|hyperscan.MultiLine) - So(string(p.Expression), ShouldEqual, "test") + So(p.Expression, ShouldEqual, "test") So(p.String(), ShouldEqual, `/test/im`) Convey("When pattern contains forward slash", func() { diff --git a/internal/ch/allocator.go b/internal/ch/allocator.go index 92808d2..ddb7d71 100644 --- a/internal/ch/allocator.go +++ b/internal/ch/allocator.go @@ -37,7 +37,7 @@ extern void chMiscFree(void *ptr); extern void *chScratchAlloc(size_t size); extern void chScratchFree(void *ptr); */ -import "C" // nolint: typecheck +import "C" type ( // The type of the callback function that will be used by Chimera to allocate more memory at runtime as required. diff --git a/internal/ch/common_test.go b/internal/ch/common_test.go index d55d8a7..413a2b5 100644 --- a/internal/ch/common_test.go +++ b/internal/ch/common_test.go @@ -1,15 +1,17 @@ -package ch +package ch_test import ( "regexp" "testing" . "github.com/smartystreets/goconvey/convey" + + "github.com/flier/gohs/internal/ch" ) func TestVersion(t *testing.T) { Convey("Given a Chimera version", t, func() { - ver := Version() + ver := ch.Version() So(ver, ShouldNotBeEmpty) diff --git a/internal/ch/compile.go b/internal/ch/compile.go index c07fd3d..d914deb 100644 --- a/internal/ch/compile.go +++ b/internal/ch/compile.go @@ -14,11 +14,10 @@ import ( import "C" // Pattern is a matching pattern. -// nolint: golint,revive,stylecheck type Pattern struct { Expression string // The expression to parse. Flags CompileFlag // Flags which modify the behaviour of the expression. - Id int // The ID number to be associated with the corresponding pattern + ID int // The ID number to be associated with the corresponding pattern } // NewPattern returns a new pattern base on expression and compile flags. @@ -155,7 +154,7 @@ func CompileExtMulti(p Patterns, mode CompileMode, info *hs.PlatformInfo, for i, pattern := range patterns { exprs[i] = C.CString(pattern.Expression) flags[i] = C.uint(pattern.Flags) - ids[i] = C.uint(pattern.Id) + ids[i] = C.uint(pattern.ID) } ret := C.ch_compile_ext_multi(cexprs, cflags, cids, C.uint(len(patterns)), C.uint(mode), diff --git a/internal/ch/runtime.go b/internal/ch/runtime.go index b7683ba..d9cbaf2 100644 --- a/internal/ch/runtime.go +++ b/internal/ch/runtime.go @@ -51,7 +51,7 @@ type Capture struct { type MatchEventHandler func(id uint, from, to uint64, flags uint, captured []*Capture, context interface{}) Callback // Type used to differentiate the errors raised with the `ErrorEventHandler` callback. -type ErrorEvent C.ch_error_event_t +type ErrorEvent C.ch_error_event_t // nolint: errname const ( // PCRE hits its match limit and reports PCRE_ERROR_MATCHLIMIT. @@ -83,7 +83,7 @@ type eventContext struct { //export matchEventCallback func matchEventCallback(id C.uint, from, to C.ulonglong, flags, size C.uint, - cap *C.capture_t, data unsafe.Pointer) C.ch_callback_t { + capture *C.capture_t, data unsafe.Pointer) C.ch_callback_t { h := (*handle.Handle)(data) ctx, ok := h.Value().(eventContext) if !ok { @@ -91,7 +91,7 @@ func matchEventCallback(id C.uint, from, to C.ulonglong, flags, size C.uint, } captured := make([]*Capture, size) - for i, c := range (*[1 << 30]C.capture_t)(unsafe.Pointer(cap))[:size:size] { + for i, c := range (*[1 << 30]C.capture_t)(unsafe.Pointer(capture))[:size:size] { if c.flags == C.CH_CAPTURE_FLAG_ACTIVE { captured[i] = &Capture{uint64(c.from), uint64(c.to), ctx.data[c.from:c.to]} } From 445e7524dd3710a4d08e742fc0a20236d0066cf7 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Wed, 24 Nov 2021 18:00:46 +0800 Subject: [PATCH 21/34] remove unused package --- go.mod | 1 - go.sum | 8 -------- vendor/modules.txt | 1 - 3 files changed, 10 deletions(-) diff --git a/go.mod b/go.mod index 3b11dc5..c52c362 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.14 require ( github.com/google/gopacket v1.1.19 github.com/gopherjs/gopherjs v0.0.0-20211111143520-d0d5ecc1a356 // indirect - github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/smartystreets/assertions v1.2.1 // indirect github.com/smartystreets/goconvey v1.7.2 golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 // indirect diff --git a/go.sum b/go.sum index 28454a1..aa659ad 100644 --- a/go.sum +++ b/go.sum @@ -138,8 +138,6 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20210918183006-daae65060a40 h1:AcvWELS0eUaNBj+uaWpwd18OSGuD9sf4KjDX4LMNbT4= -github.com/gopherjs/gopherjs v0.0.0-20210918183006-daae65060a40/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI= github.com/gopherjs/gopherjs v0.0.0-20211111143520-d0d5ecc1a356 h1:d3wWSjdOuGrMHa8+Tvw3z9EGPzATpzVq1BmGK3+IyeU= github.com/gopherjs/gopherjs v0.0.0-20211111143520-d0d5ecc1a356/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -211,11 +209,9 @@ github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/assertions v1.2.1 h1:bKNHfEv7tSIjZ8JbKaFjzFINljxG4lzZvmHUnElzOIg= github.com/smartystreets/assertions v1.2.1/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrxfxaaSlJazyFLYVFx8= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= @@ -330,8 +326,6 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210917221730-978cfadd31cf h1:R150MpwJIv1MpS0N/pc+NhTM8ajzvlmxlY5OYsrevXQ= -golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 h1:0qxwC5n+ttVOINCBeRHO0nq9X7uy8SDsPoi5OaCdIEI= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -401,8 +395,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210921065528-437939a70204 h1:JJhkWtBuTQKyz2bd5WG9H8iUsJRU3En/KRfN8B2RnDs= -golang.org/x/sys v0.0.0-20210921065528-437939a70204/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab h1:rfJ1bsoJQQIAoAxTxB7bme+vHrNkRw8CqfsYh9w54cw= golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/vendor/modules.txt b/vendor/modules.txt index defb2ba..9d7e457 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -7,7 +7,6 @@ github.com/google/gopacket/pcap ## explicit github.com/gopherjs/gopherjs/js # github.com/jtolds/gls v4.20.0+incompatible -## explicit github.com/jtolds/gls # github.com/smartystreets/assertions v1.2.1 ## explicit From 1c6de84d49a6fd7f2b72fda08a208ea2ec11394d Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sun, 10 Apr 2022 11:26:50 +0800 Subject: [PATCH 22/34] use chimera build tag to enable chimera support --- .github/workflows/ci.yml | 12 ++++++++---- chimera/api.go | 3 +++ chimera/api_test.go | 3 +++ chimera/block.go | 3 +++ chimera/common.go | 3 +++ chimera/common_test.go | 3 +++ chimera/compile.go | 3 +++ chimera/compile_test.go | 3 +++ chimera/doc.go | 3 +++ chimera/example_api_test.go | 3 +++ chimera/example_compile_test.go | 3 +++ chimera/example_runtime_test.go | 3 +++ chimera/pattern.go | 3 +++ chimera/pattern_test.go | 3 +++ chimera/runtime.go | 3 +++ chimera/runtime_test.go | 3 +++ chimera/scratch.go | 3 +++ internal/ch/allocator.go | 3 +++ internal/ch/allocator_test.go | 3 +++ internal/ch/common.go | 3 +++ internal/ch/common_test.go | 3 +++ internal/ch/compile.go | 3 +++ internal/ch/compile_test.go | 3 +++ internal/ch/error.go | 3 +++ internal/ch/error_v54.go | 4 ++-- internal/ch/link.go | 3 +++ internal/ch/runtime.go | 3 +++ internal/ch/runtime_test.go | 3 +++ internal/ch/scratch.go | 3 +++ internal/ch/scratch_test.go | 3 +++ 30 files changed, 94 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf4e315..c61aa7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04, ubuntu-18.04, macos-latest] - go: [1.17.x, 1.16.x, 1.15.x, 1.14.x] + go: [1.18.x 1.17.x, 1.16.x, 1.15.x] name: Go ${{ matrix.go }} tests @ ${{ matrix.os }} for hyperscan ${{ matrix.hyperscan }} runs-on: ${{ matrix.os }} steps: @@ -71,14 +71,18 @@ jobs: - os: macos-latest hyperscan: 5.4.0 pcre: 8.45 - build_flags: -tags hyperscan_v54 + build_flags: -tags hyperscan_v54,chimera + chimera: true - os: macos-latest hyperscan: 5.2.1 pcre: 8.45 + build_flags: -tags chimera + chimera: true - os: ubuntu-20.04 hyperscan: 5.4.0 pcre: 8.45 - build_flags: -tags hyperscan_v54 + build_flags: -tags hyperscan_v54,chimera + chimera: true coverage: true - os: ubuntu-20.04 hyperscan: 5.2.1 @@ -165,7 +169,7 @@ jobs: go test -race -v ${{ matrix.build_flags }} ./internal/hs/... ./hyperscan/... - name: Test Chimera API - if: matrix.coverage + if: matrix.chimera run: | go test -v ${{ matrix.build_flags }} ./internal/ch/... ./chimera/... go test -race -v ${{ matrix.build_flags }} ./internal/ch/... ./chimera/... diff --git a/chimera/api.go b/chimera/api.go index 3e730b7..97ef1cb 100644 --- a/chimera/api.go +++ b/chimera/api.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera import ( diff --git a/chimera/api_test.go b/chimera/api_test.go index 6d5d770..ee391fc 100644 --- a/chimera/api_test.go +++ b/chimera/api_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera_test import ( diff --git a/chimera/block.go b/chimera/block.go index 91a5fdd..b16967f 100644 --- a/chimera/block.go +++ b/chimera/block.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera import ( diff --git a/chimera/common.go b/chimera/common.go index 6f1a691..f55abc7 100644 --- a/chimera/common.go +++ b/chimera/common.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera import ( diff --git a/chimera/common_test.go b/chimera/common_test.go index fc0ae96..eef5ac1 100644 --- a/chimera/common_test.go +++ b/chimera/common_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera_test import ( diff --git a/chimera/compile.go b/chimera/compile.go index 8ce4a8b..d4c47f8 100644 --- a/chimera/compile.go +++ b/chimera/compile.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera import ( diff --git a/chimera/compile_test.go b/chimera/compile_test.go index c62c784..dd9e1ae 100644 --- a/chimera/compile_test.go +++ b/chimera/compile_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera_test import ( diff --git a/chimera/doc.go b/chimera/doc.go index f3898c3..51227e5 100644 --- a/chimera/doc.go +++ b/chimera/doc.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + // Chimera is a software regular expression matching engine that is a hybrid of Hyperscan and PCRE. // The design goals of Chimera are to fully support PCRE syntax as well as to // take advantage of the high performance nature of Hyperscan. diff --git a/chimera/example_api_test.go b/chimera/example_api_test.go index 3102152..4c70785 100644 --- a/chimera/example_api_test.go +++ b/chimera/example_api_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera_test import ( diff --git a/chimera/example_compile_test.go b/chimera/example_compile_test.go index a0d039e..6b12857 100644 --- a/chimera/example_compile_test.go +++ b/chimera/example_compile_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera_test import ( diff --git a/chimera/example_runtime_test.go b/chimera/example_runtime_test.go index 5d71be5..62c4b99 100644 --- a/chimera/example_runtime_test.go +++ b/chimera/example_runtime_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera_test import ( diff --git a/chimera/pattern.go b/chimera/pattern.go index 0d4d916..84fcb0d 100644 --- a/chimera/pattern.go +++ b/chimera/pattern.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera import ( diff --git a/chimera/pattern_test.go b/chimera/pattern_test.go index 2e87fc0..bae37c2 100644 --- a/chimera/pattern_test.go +++ b/chimera/pattern_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera_test import ( diff --git a/chimera/runtime.go b/chimera/runtime.go index 1b4b885..42eac20 100644 --- a/chimera/runtime.go +++ b/chimera/runtime.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera import ( diff --git a/chimera/runtime_test.go b/chimera/runtime_test.go index 7801be9..d769872 100644 --- a/chimera/runtime_test.go +++ b/chimera/runtime_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera_test import ( diff --git a/chimera/scratch.go b/chimera/scratch.go index dbbfd0f..a6e1299 100644 --- a/chimera/scratch.go +++ b/chimera/scratch.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package chimera import ( diff --git a/internal/ch/allocator.go b/internal/ch/allocator.go index 92808d2..716fb5a 100644 --- a/internal/ch/allocator.go +++ b/internal/ch/allocator.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch import "unsafe" diff --git a/internal/ch/allocator_test.go b/internal/ch/allocator_test.go index e04c80a..6065805 100644 --- a/internal/ch/allocator_test.go +++ b/internal/ch/allocator_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch_test import ( diff --git a/internal/ch/common.go b/internal/ch/common.go index 730fc2c..5e8ba69 100644 --- a/internal/ch/common.go +++ b/internal/ch/common.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch import ( diff --git a/internal/ch/common_test.go b/internal/ch/common_test.go index d55d8a7..a97b3ea 100644 --- a/internal/ch/common_test.go +++ b/internal/ch/common_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch import ( diff --git a/internal/ch/compile.go b/internal/ch/compile.go index b4ba848..3c2c7a6 100644 --- a/internal/ch/compile.go +++ b/internal/ch/compile.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch import ( diff --git a/internal/ch/compile_test.go b/internal/ch/compile_test.go index 500b8f5..4849b7b 100644 --- a/internal/ch/compile_test.go +++ b/internal/ch/compile_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch_test import ( diff --git a/internal/ch/error.go b/internal/ch/error.go index 52d6b28..61c9f66 100644 --- a/internal/ch/error.go +++ b/internal/ch/error.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch // #include diff --git a/internal/ch/error_v54.go b/internal/ch/error_v54.go index 17e9086..439bf78 100644 --- a/internal/ch/error_v54.go +++ b/internal/ch/error_v54.go @@ -1,5 +1,5 @@ -//go:build hyperscan_v54 -// +build hyperscan_v54 +//go:build hyperscan_v54 && chimera +// +build hyperscan_v54,chimera package ch diff --git a/internal/ch/link.go b/internal/ch/link.go index 13bce5e..9b6a3b4 100644 --- a/internal/ch/link.go +++ b/internal/ch/link.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch /* diff --git a/internal/ch/runtime.go b/internal/ch/runtime.go index 5410d9a..4a99be4 100644 --- a/internal/ch/runtime.go +++ b/internal/ch/runtime.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch import ( diff --git a/internal/ch/runtime_test.go b/internal/ch/runtime_test.go index 7df4f12..4807286 100644 --- a/internal/ch/runtime_test.go +++ b/internal/ch/runtime_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch_test import ( diff --git a/internal/ch/scratch.go b/internal/ch/scratch.go index e08f268..c23a6a2 100644 --- a/internal/ch/scratch.go +++ b/internal/ch/scratch.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch // #include diff --git a/internal/ch/scratch_test.go b/internal/ch/scratch_test.go index 40b18cd..5ef62b1 100644 --- a/internal/ch/scratch_test.go +++ b/internal/ch/scratch_test.go @@ -1,3 +1,6 @@ +//go:build chimera +// +build chimera + package ch_test import ( From db4a452fa51db85d6290803b61b3fc91b9874d76 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sun, 10 Apr 2022 11:35:36 +0800 Subject: [PATCH 23/34] upgrade dependencies --- go.mod | 6 +- go.sum | 19 +- vendor/github.com/gopherjs/gopherjs/js/js.go | 8 +- .../golang.org/x/sys/windows/exec_windows.go | 37 +- vendor/golang.org/x/sys/windows/mksyscall.go | 2 +- .../x/sys/windows/setupapi_windows.go | 1425 +++++++++++++++++ .../x/sys/windows/setupapierrors_windows.go | 100 -- .../x/sys/windows/syscall_windows.go | 47 +- .../golang.org/x/sys/windows/types_windows.go | 60 +- .../x/sys/windows/zsyscall_windows.go | 331 +++- vendor/modules.txt | 6 +- 11 files changed, 1879 insertions(+), 162 deletions(-) create mode 100644 vendor/golang.org/x/sys/windows/setupapi_windows.go delete mode 100644 vendor/golang.org/x/sys/windows/setupapierrors_windows.go diff --git a/go.mod b/go.mod index c52c362..1f949e1 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.14 require ( github.com/google/gopacket v1.1.19 - github.com/gopherjs/gopherjs v0.0.0-20211111143520-d0d5ecc1a356 // indirect + github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96 // indirect github.com/smartystreets/assertions v1.2.1 // indirect github.com/smartystreets/goconvey v1.7.2 - golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 // indirect - golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab // indirect + golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 // indirect + golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f // indirect ) diff --git a/go.sum b/go.sum index aa659ad..b94aff1 100644 --- a/go.sum +++ b/go.sum @@ -138,8 +138,8 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20211111143520-d0d5ecc1a356 h1:d3wWSjdOuGrMHa8+Tvw3z9EGPzATpzVq1BmGK3+IyeU= -github.com/gopherjs/gopherjs v0.0.0-20211111143520-d0d5ecc1a356/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI= +github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96 h1:QJq7UBOuoynsywLk+aC75rC2Cbi2+lQRDaLaizhA+fA= +github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -208,6 +208,7 @@ github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxr github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/assertions v1.2.1 h1:bKNHfEv7tSIjZ8JbKaFjzFINljxG4lzZvmHUnElzOIg= @@ -326,8 +327,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 h1:0qxwC5n+ttVOINCBeRHO0nq9X7uy8SDsPoi5OaCdIEI= -golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 h1:EN5+DfgmRMvRUrMGERW2gQl3Vc+Z7ZMnI/xdEpPSf0c= +golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -364,6 +365,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -391,13 +393,14 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab h1:rfJ1bsoJQQIAoAxTxB7bme+vHrNkRw8CqfsYh9w54cw= -golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f h1:8w7RhxzTVgUzw/AH/9mUV5q0vMgy40SQRursCcfmkCw= +golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -405,7 +408,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/vendor/github.com/gopherjs/gopherjs/js/js.go b/vendor/github.com/gopherjs/gopherjs/js/js.go index 3fbf1d8..2b5938b 100644 --- a/vendor/github.com/gopherjs/gopherjs/js/js.go +++ b/vendor/github.com/gopherjs/gopherjs/js/js.go @@ -97,7 +97,13 @@ func (err *Error) Stack() string { // Global gives JavaScript's global object ("window" for browsers and "GLOBAL" for Node.js). var Global *Object -// Module gives the value of the "module" variable set by Node.js. Hint: Set a module export with 'js.Module.Get("exports").Set("exportName", ...)'. +// Module gives the value of the "module" variable set by Node.js. Hint: Set a +// module export with 'js.Module.Get("exports").Set("exportName", ...)'. +// +// Note that js.Module is only defined in runtimes which support CommonJS +// modules (https://nodejs.org/api/modules.html). NodeJS supports it natively, +// but in browsers it can only be used if GopherJS output is passed through a +// bundler which implements CommonJS (for example, webpack or esbuild). var Module *Object // Undefined gives the JavaScript value "undefined". diff --git a/vendor/golang.org/x/sys/windows/exec_windows.go b/vendor/golang.org/x/sys/windows/exec_windows.go index 7a11e83..855698b 100644 --- a/vendor/golang.org/x/sys/windows/exec_windows.go +++ b/vendor/golang.org/x/sys/windows/exec_windows.go @@ -9,8 +9,6 @@ package windows import ( errorspkg "errors" "unsafe" - - "golang.org/x/sys/internal/unsafeheader" ) // EscapeArg rewrites command line argument s as prescribed @@ -147,8 +145,12 @@ func NewProcThreadAttributeList(maxAttrCount uint32) (*ProcThreadAttributeListCo } return nil, err } + alloc, err := LocalAlloc(LMEM_FIXED, uint32(size)) + if err != nil { + return nil, err + } // size is guaranteed to be ≥1 by InitializeProcThreadAttributeList. - al := &ProcThreadAttributeListContainer{data: (*ProcThreadAttributeList)(unsafe.Pointer(&make([]byte, size)[0]))} + al := &ProcThreadAttributeListContainer{data: (*ProcThreadAttributeList)(unsafe.Pointer(alloc))} err = initializeProcThreadAttributeList(al.data, maxAttrCount, 0, &size) if err != nil { return nil, err @@ -157,36 +159,17 @@ func NewProcThreadAttributeList(maxAttrCount uint32) (*ProcThreadAttributeListCo } // Update modifies the ProcThreadAttributeList using UpdateProcThreadAttribute. -// Note that the value passed to this function will be copied into memory -// allocated by LocalAlloc, the contents of which should not contain any -// Go-managed pointers, even if the passed value itself is a Go-managed -// pointer. func (al *ProcThreadAttributeListContainer) Update(attribute uintptr, value unsafe.Pointer, size uintptr) error { - alloc, err := LocalAlloc(LMEM_FIXED, uint32(size)) - if err != nil { - return err - } - var src, dst []byte - hdr := (*unsafeheader.Slice)(unsafe.Pointer(&src)) - hdr.Data = value - hdr.Cap = int(size) - hdr.Len = int(size) - hdr = (*unsafeheader.Slice)(unsafe.Pointer(&dst)) - hdr.Data = unsafe.Pointer(alloc) - hdr.Cap = int(size) - hdr.Len = int(size) - copy(dst, src) - al.heapAllocations = append(al.heapAllocations, alloc) - return updateProcThreadAttribute(al.data, 0, attribute, unsafe.Pointer(alloc), size, nil, nil) + al.pointers = append(al.pointers, value) + return updateProcThreadAttribute(al.data, 0, attribute, value, size, nil, nil) } // Delete frees ProcThreadAttributeList's resources. func (al *ProcThreadAttributeListContainer) Delete() { deleteProcThreadAttributeList(al.data) - for i := range al.heapAllocations { - LocalFree(Handle(al.heapAllocations[i])) - } - al.heapAllocations = nil + LocalFree(Handle(unsafe.Pointer(al.data))) + al.data = nil + al.pointers = nil } // List returns the actual ProcThreadAttributeList to be passed to StartupInfoEx. diff --git a/vendor/golang.org/x/sys/windows/mksyscall.go b/vendor/golang.org/x/sys/windows/mksyscall.go index 6102910..8563f79 100644 --- a/vendor/golang.org/x/sys/windows/mksyscall.go +++ b/vendor/golang.org/x/sys/windows/mksyscall.go @@ -7,4 +7,4 @@ package windows -//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go eventlog.go service.go syscall_windows.go security_windows.go +//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go eventlog.go service.go syscall_windows.go security_windows.go setupapi_windows.go diff --git a/vendor/golang.org/x/sys/windows/setupapi_windows.go b/vendor/golang.org/x/sys/windows/setupapi_windows.go new file mode 100644 index 0000000..14027da --- /dev/null +++ b/vendor/golang.org/x/sys/windows/setupapi_windows.go @@ -0,0 +1,1425 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows + +import ( + "encoding/binary" + "errors" + "fmt" + "runtime" + "strings" + "syscall" + "unsafe" +) + +// This file contains functions that wrap SetupAPI.dll and CfgMgr32.dll, +// core system functions for managing hardware devices, drivers, and the PnP tree. +// Information about these APIs can be found at: +// https://docs.microsoft.com/en-us/windows-hardware/drivers/install/setupapi +// https://docs.microsoft.com/en-us/windows/win32/devinst/cfgmgr32- + +const ( + ERROR_EXPECTED_SECTION_NAME Errno = 0x20000000 | 0xC0000000 | 0 + ERROR_BAD_SECTION_NAME_LINE Errno = 0x20000000 | 0xC0000000 | 1 + ERROR_SECTION_NAME_TOO_LONG Errno = 0x20000000 | 0xC0000000 | 2 + ERROR_GENERAL_SYNTAX Errno = 0x20000000 | 0xC0000000 | 3 + ERROR_WRONG_INF_STYLE Errno = 0x20000000 | 0xC0000000 | 0x100 + ERROR_SECTION_NOT_FOUND Errno = 0x20000000 | 0xC0000000 | 0x101 + ERROR_LINE_NOT_FOUND Errno = 0x20000000 | 0xC0000000 | 0x102 + ERROR_NO_BACKUP Errno = 0x20000000 | 0xC0000000 | 0x103 + ERROR_NO_ASSOCIATED_CLASS Errno = 0x20000000 | 0xC0000000 | 0x200 + ERROR_CLASS_MISMATCH Errno = 0x20000000 | 0xC0000000 | 0x201 + ERROR_DUPLICATE_FOUND Errno = 0x20000000 | 0xC0000000 | 0x202 + ERROR_NO_DRIVER_SELECTED Errno = 0x20000000 | 0xC0000000 | 0x203 + ERROR_KEY_DOES_NOT_EXIST Errno = 0x20000000 | 0xC0000000 | 0x204 + ERROR_INVALID_DEVINST_NAME Errno = 0x20000000 | 0xC0000000 | 0x205 + ERROR_INVALID_CLASS Errno = 0x20000000 | 0xC0000000 | 0x206 + ERROR_DEVINST_ALREADY_EXISTS Errno = 0x20000000 | 0xC0000000 | 0x207 + ERROR_DEVINFO_NOT_REGISTERED Errno = 0x20000000 | 0xC0000000 | 0x208 + ERROR_INVALID_REG_PROPERTY Errno = 0x20000000 | 0xC0000000 | 0x209 + ERROR_NO_INF Errno = 0x20000000 | 0xC0000000 | 0x20A + ERROR_NO_SUCH_DEVINST Errno = 0x20000000 | 0xC0000000 | 0x20B + ERROR_CANT_LOAD_CLASS_ICON Errno = 0x20000000 | 0xC0000000 | 0x20C + ERROR_INVALID_CLASS_INSTALLER Errno = 0x20000000 | 0xC0000000 | 0x20D + ERROR_DI_DO_DEFAULT Errno = 0x20000000 | 0xC0000000 | 0x20E + ERROR_DI_NOFILECOPY Errno = 0x20000000 | 0xC0000000 | 0x20F + ERROR_INVALID_HWPROFILE Errno = 0x20000000 | 0xC0000000 | 0x210 + ERROR_NO_DEVICE_SELECTED Errno = 0x20000000 | 0xC0000000 | 0x211 + ERROR_DEVINFO_LIST_LOCKED Errno = 0x20000000 | 0xC0000000 | 0x212 + ERROR_DEVINFO_DATA_LOCKED Errno = 0x20000000 | 0xC0000000 | 0x213 + ERROR_DI_BAD_PATH Errno = 0x20000000 | 0xC0000000 | 0x214 + ERROR_NO_CLASSINSTALL_PARAMS Errno = 0x20000000 | 0xC0000000 | 0x215 + ERROR_FILEQUEUE_LOCKED Errno = 0x20000000 | 0xC0000000 | 0x216 + ERROR_BAD_SERVICE_INSTALLSECT Errno = 0x20000000 | 0xC0000000 | 0x217 + ERROR_NO_CLASS_DRIVER_LIST Errno = 0x20000000 | 0xC0000000 | 0x218 + ERROR_NO_ASSOCIATED_SERVICE Errno = 0x20000000 | 0xC0000000 | 0x219 + ERROR_NO_DEFAULT_DEVICE_INTERFACE Errno = 0x20000000 | 0xC0000000 | 0x21A + ERROR_DEVICE_INTERFACE_ACTIVE Errno = 0x20000000 | 0xC0000000 | 0x21B + ERROR_DEVICE_INTERFACE_REMOVED Errno = 0x20000000 | 0xC0000000 | 0x21C + ERROR_BAD_INTERFACE_INSTALLSECT Errno = 0x20000000 | 0xC0000000 | 0x21D + ERROR_NO_SUCH_INTERFACE_CLASS Errno = 0x20000000 | 0xC0000000 | 0x21E + ERROR_INVALID_REFERENCE_STRING Errno = 0x20000000 | 0xC0000000 | 0x21F + ERROR_INVALID_MACHINENAME Errno = 0x20000000 | 0xC0000000 | 0x220 + ERROR_REMOTE_COMM_FAILURE Errno = 0x20000000 | 0xC0000000 | 0x221 + ERROR_MACHINE_UNAVAILABLE Errno = 0x20000000 | 0xC0000000 | 0x222 + ERROR_NO_CONFIGMGR_SERVICES Errno = 0x20000000 | 0xC0000000 | 0x223 + ERROR_INVALID_PROPPAGE_PROVIDER Errno = 0x20000000 | 0xC0000000 | 0x224 + ERROR_NO_SUCH_DEVICE_INTERFACE Errno = 0x20000000 | 0xC0000000 | 0x225 + ERROR_DI_POSTPROCESSING_REQUIRED Errno = 0x20000000 | 0xC0000000 | 0x226 + ERROR_INVALID_COINSTALLER Errno = 0x20000000 | 0xC0000000 | 0x227 + ERROR_NO_COMPAT_DRIVERS Errno = 0x20000000 | 0xC0000000 | 0x228 + ERROR_NO_DEVICE_ICON Errno = 0x20000000 | 0xC0000000 | 0x229 + ERROR_INVALID_INF_LOGCONFIG Errno = 0x20000000 | 0xC0000000 | 0x22A + ERROR_DI_DONT_INSTALL Errno = 0x20000000 | 0xC0000000 | 0x22B + ERROR_INVALID_FILTER_DRIVER Errno = 0x20000000 | 0xC0000000 | 0x22C + ERROR_NON_WINDOWS_NT_DRIVER Errno = 0x20000000 | 0xC0000000 | 0x22D + ERROR_NON_WINDOWS_DRIVER Errno = 0x20000000 | 0xC0000000 | 0x22E + ERROR_NO_CATALOG_FOR_OEM_INF Errno = 0x20000000 | 0xC0000000 | 0x22F + ERROR_DEVINSTALL_QUEUE_NONNATIVE Errno = 0x20000000 | 0xC0000000 | 0x230 + ERROR_NOT_DISABLEABLE Errno = 0x20000000 | 0xC0000000 | 0x231 + ERROR_CANT_REMOVE_DEVINST Errno = 0x20000000 | 0xC0000000 | 0x232 + ERROR_INVALID_TARGET Errno = 0x20000000 | 0xC0000000 | 0x233 + ERROR_DRIVER_NONNATIVE Errno = 0x20000000 | 0xC0000000 | 0x234 + ERROR_IN_WOW64 Errno = 0x20000000 | 0xC0000000 | 0x235 + ERROR_SET_SYSTEM_RESTORE_POINT Errno = 0x20000000 | 0xC0000000 | 0x236 + ERROR_SCE_DISABLED Errno = 0x20000000 | 0xC0000000 | 0x238 + ERROR_UNKNOWN_EXCEPTION Errno = 0x20000000 | 0xC0000000 | 0x239 + ERROR_PNP_REGISTRY_ERROR Errno = 0x20000000 | 0xC0000000 | 0x23A + ERROR_REMOTE_REQUEST_UNSUPPORTED Errno = 0x20000000 | 0xC0000000 | 0x23B + ERROR_NOT_AN_INSTALLED_OEM_INF Errno = 0x20000000 | 0xC0000000 | 0x23C + ERROR_INF_IN_USE_BY_DEVICES Errno = 0x20000000 | 0xC0000000 | 0x23D + ERROR_DI_FUNCTION_OBSOLETE Errno = 0x20000000 | 0xC0000000 | 0x23E + ERROR_NO_AUTHENTICODE_CATALOG Errno = 0x20000000 | 0xC0000000 | 0x23F + ERROR_AUTHENTICODE_DISALLOWED Errno = 0x20000000 | 0xC0000000 | 0x240 + ERROR_AUTHENTICODE_TRUSTED_PUBLISHER Errno = 0x20000000 | 0xC0000000 | 0x241 + ERROR_AUTHENTICODE_TRUST_NOT_ESTABLISHED Errno = 0x20000000 | 0xC0000000 | 0x242 + ERROR_AUTHENTICODE_PUBLISHER_NOT_TRUSTED Errno = 0x20000000 | 0xC0000000 | 0x243 + ERROR_SIGNATURE_OSATTRIBUTE_MISMATCH Errno = 0x20000000 | 0xC0000000 | 0x244 + ERROR_ONLY_VALIDATE_VIA_AUTHENTICODE Errno = 0x20000000 | 0xC0000000 | 0x245 + ERROR_DEVICE_INSTALLER_NOT_READY Errno = 0x20000000 | 0xC0000000 | 0x246 + ERROR_DRIVER_STORE_ADD_FAILED Errno = 0x20000000 | 0xC0000000 | 0x247 + ERROR_DEVICE_INSTALL_BLOCKED Errno = 0x20000000 | 0xC0000000 | 0x248 + ERROR_DRIVER_INSTALL_BLOCKED Errno = 0x20000000 | 0xC0000000 | 0x249 + ERROR_WRONG_INF_TYPE Errno = 0x20000000 | 0xC0000000 | 0x24A + ERROR_FILE_HASH_NOT_IN_CATALOG Errno = 0x20000000 | 0xC0000000 | 0x24B + ERROR_DRIVER_STORE_DELETE_FAILED Errno = 0x20000000 | 0xC0000000 | 0x24C + ERROR_UNRECOVERABLE_STACK_OVERFLOW Errno = 0x20000000 | 0xC0000000 | 0x300 + EXCEPTION_SPAPI_UNRECOVERABLE_STACK_OVERFLOW Errno = ERROR_UNRECOVERABLE_STACK_OVERFLOW + ERROR_NO_DEFAULT_INTERFACE_DEVICE Errno = ERROR_NO_DEFAULT_DEVICE_INTERFACE + ERROR_INTERFACE_DEVICE_ACTIVE Errno = ERROR_DEVICE_INTERFACE_ACTIVE + ERROR_INTERFACE_DEVICE_REMOVED Errno = ERROR_DEVICE_INTERFACE_REMOVED + ERROR_NO_SUCH_INTERFACE_DEVICE Errno = ERROR_NO_SUCH_DEVICE_INTERFACE +) + +const ( + MAX_DEVICE_ID_LEN = 200 + MAX_DEVNODE_ID_LEN = MAX_DEVICE_ID_LEN + MAX_GUID_STRING_LEN = 39 // 38 chars + terminator null + MAX_CLASS_NAME_LEN = 32 + MAX_PROFILE_LEN = 80 + MAX_CONFIG_VALUE = 9999 + MAX_INSTANCE_VALUE = 9999 + CONFIGMG_VERSION = 0x0400 +) + +// Maximum string length constants +const ( + LINE_LEN = 256 // Windows 9x-compatible maximum for displayable strings coming from a device INF. + MAX_INF_STRING_LENGTH = 4096 // Actual maximum size of an INF string (including string substitutions). + MAX_INF_SECTION_NAME_LENGTH = 255 // For Windows 9x compatibility, INF section names should be constrained to 32 characters. + MAX_TITLE_LEN = 60 + MAX_INSTRUCTION_LEN = 256 + MAX_LABEL_LEN = 30 + MAX_SERVICE_NAME_LEN = 256 + MAX_SUBTITLE_LEN = 256 +) + +const ( + // SP_MAX_MACHINENAME_LENGTH defines maximum length of a machine name in the format expected by ConfigMgr32 CM_Connect_Machine (i.e., "\\\\MachineName\0"). + SP_MAX_MACHINENAME_LENGTH = MAX_PATH + 3 +) + +// HSPFILEQ is type for setup file queue +type HSPFILEQ uintptr + +// DevInfo holds reference to device information set +type DevInfo Handle + +// DEVINST is a handle usually recognized by cfgmgr32 APIs +type DEVINST uint32 + +// DevInfoData is a device information structure (references a device instance that is a member of a device information set) +type DevInfoData struct { + size uint32 + ClassGUID GUID + DevInst DEVINST + _ uintptr +} + +// DevInfoListDetailData is a structure for detailed information on a device information set (used for SetupDiGetDeviceInfoListDetail which supersedes the functionality of SetupDiGetDeviceInfoListClass). +type DevInfoListDetailData struct { + size uint32 // Use unsafeSizeOf method + ClassGUID GUID + RemoteMachineHandle Handle + remoteMachineName [SP_MAX_MACHINENAME_LENGTH]uint16 +} + +func (*DevInfoListDetailData) unsafeSizeOf() uint32 { + if unsafe.Sizeof(uintptr(0)) == 4 { + // Windows declares this with pshpack1.h + return uint32(unsafe.Offsetof(DevInfoListDetailData{}.remoteMachineName) + unsafe.Sizeof(DevInfoListDetailData{}.remoteMachineName)) + } + return uint32(unsafe.Sizeof(DevInfoListDetailData{})) +} + +func (data *DevInfoListDetailData) RemoteMachineName() string { + return UTF16ToString(data.remoteMachineName[:]) +} + +func (data *DevInfoListDetailData) SetRemoteMachineName(remoteMachineName string) error { + str, err := UTF16FromString(remoteMachineName) + if err != nil { + return err + } + copy(data.remoteMachineName[:], str) + return nil +} + +// DI_FUNCTION is function type for device installer +type DI_FUNCTION uint32 + +const ( + DIF_SELECTDEVICE DI_FUNCTION = 0x00000001 + DIF_INSTALLDEVICE DI_FUNCTION = 0x00000002 + DIF_ASSIGNRESOURCES DI_FUNCTION = 0x00000003 + DIF_PROPERTIES DI_FUNCTION = 0x00000004 + DIF_REMOVE DI_FUNCTION = 0x00000005 + DIF_FIRSTTIMESETUP DI_FUNCTION = 0x00000006 + DIF_FOUNDDEVICE DI_FUNCTION = 0x00000007 + DIF_SELECTCLASSDRIVERS DI_FUNCTION = 0x00000008 + DIF_VALIDATECLASSDRIVERS DI_FUNCTION = 0x00000009 + DIF_INSTALLCLASSDRIVERS DI_FUNCTION = 0x0000000A + DIF_CALCDISKSPACE DI_FUNCTION = 0x0000000B + DIF_DESTROYPRIVATEDATA DI_FUNCTION = 0x0000000C + DIF_VALIDATEDRIVER DI_FUNCTION = 0x0000000D + DIF_DETECT DI_FUNCTION = 0x0000000F + DIF_INSTALLWIZARD DI_FUNCTION = 0x00000010 + DIF_DESTROYWIZARDDATA DI_FUNCTION = 0x00000011 + DIF_PROPERTYCHANGE DI_FUNCTION = 0x00000012 + DIF_ENABLECLASS DI_FUNCTION = 0x00000013 + DIF_DETECTVERIFY DI_FUNCTION = 0x00000014 + DIF_INSTALLDEVICEFILES DI_FUNCTION = 0x00000015 + DIF_UNREMOVE DI_FUNCTION = 0x00000016 + DIF_SELECTBESTCOMPATDRV DI_FUNCTION = 0x00000017 + DIF_ALLOW_INSTALL DI_FUNCTION = 0x00000018 + DIF_REGISTERDEVICE DI_FUNCTION = 0x00000019 + DIF_NEWDEVICEWIZARD_PRESELECT DI_FUNCTION = 0x0000001A + DIF_NEWDEVICEWIZARD_SELECT DI_FUNCTION = 0x0000001B + DIF_NEWDEVICEWIZARD_PREANALYZE DI_FUNCTION = 0x0000001C + DIF_NEWDEVICEWIZARD_POSTANALYZE DI_FUNCTION = 0x0000001D + DIF_NEWDEVICEWIZARD_FINISHINSTALL DI_FUNCTION = 0x0000001E + DIF_INSTALLINTERFACES DI_FUNCTION = 0x00000020 + DIF_DETECTCANCEL DI_FUNCTION = 0x00000021 + DIF_REGISTER_COINSTALLERS DI_FUNCTION = 0x00000022 + DIF_ADDPROPERTYPAGE_ADVANCED DI_FUNCTION = 0x00000023 + DIF_ADDPROPERTYPAGE_BASIC DI_FUNCTION = 0x00000024 + DIF_TROUBLESHOOTER DI_FUNCTION = 0x00000026 + DIF_POWERMESSAGEWAKE DI_FUNCTION = 0x00000027 + DIF_ADDREMOTEPROPERTYPAGE_ADVANCED DI_FUNCTION = 0x00000028 + DIF_UPDATEDRIVER_UI DI_FUNCTION = 0x00000029 + DIF_FINISHINSTALL_ACTION DI_FUNCTION = 0x0000002A +) + +// DevInstallParams is device installation parameters structure (associated with a particular device information element, or globally with a device information set) +type DevInstallParams struct { + size uint32 + Flags DI_FLAGS + FlagsEx DI_FLAGSEX + hwndParent uintptr + InstallMsgHandler uintptr + InstallMsgHandlerContext uintptr + FileQueue HSPFILEQ + _ uintptr + _ uint32 + driverPath [MAX_PATH]uint16 +} + +func (params *DevInstallParams) DriverPath() string { + return UTF16ToString(params.driverPath[:]) +} + +func (params *DevInstallParams) SetDriverPath(driverPath string) error { + str, err := UTF16FromString(driverPath) + if err != nil { + return err + } + copy(params.driverPath[:], str) + return nil +} + +// DI_FLAGS is SP_DEVINSTALL_PARAMS.Flags values +type DI_FLAGS uint32 + +const ( + // Flags for choosing a device + DI_SHOWOEM DI_FLAGS = 0x00000001 // support Other... button + DI_SHOWCOMPAT DI_FLAGS = 0x00000002 // show compatibility list + DI_SHOWCLASS DI_FLAGS = 0x00000004 // show class list + DI_SHOWALL DI_FLAGS = 0x00000007 // both class & compat list shown + DI_NOVCP DI_FLAGS = 0x00000008 // don't create a new copy queue--use caller-supplied FileQueue + DI_DIDCOMPAT DI_FLAGS = 0x00000010 // Searched for compatible devices + DI_DIDCLASS DI_FLAGS = 0x00000020 // Searched for class devices + DI_AUTOASSIGNRES DI_FLAGS = 0x00000040 // No UI for resources if possible + + // Flags returned by DiInstallDevice to indicate need to reboot/restart + DI_NEEDRESTART DI_FLAGS = 0x00000080 // Reboot required to take effect + DI_NEEDREBOOT DI_FLAGS = 0x00000100 // "" + + // Flags for device installation + DI_NOBROWSE DI_FLAGS = 0x00000200 // no Browse... in InsertDisk + + // Flags set by DiBuildDriverInfoList + DI_MULTMFGS DI_FLAGS = 0x00000400 // Set if multiple manufacturers in class driver list + + // Flag indicates that device is disabled + DI_DISABLED DI_FLAGS = 0x00000800 // Set if device disabled + + // Flags for Device/Class Properties + DI_GENERALPAGE_ADDED DI_FLAGS = 0x00001000 + DI_RESOURCEPAGE_ADDED DI_FLAGS = 0x00002000 + + // Flag to indicate the setting properties for this Device (or class) caused a change so the Dev Mgr UI probably needs to be updated. + DI_PROPERTIES_CHANGE DI_FLAGS = 0x00004000 + + // Flag to indicate that the sorting from the INF file should be used. + DI_INF_IS_SORTED DI_FLAGS = 0x00008000 + + // Flag to indicate that only the the INF specified by SP_DEVINSTALL_PARAMS.DriverPath should be searched. + DI_ENUMSINGLEINF DI_FLAGS = 0x00010000 + + // Flag that prevents ConfigMgr from removing/re-enumerating devices during device + // registration, installation, and deletion. + DI_DONOTCALLCONFIGMG DI_FLAGS = 0x00020000 + + // The following flag can be used to install a device disabled + DI_INSTALLDISABLED DI_FLAGS = 0x00040000 + + // Flag that causes SetupDiBuildDriverInfoList to build a device's compatible driver + // list from its existing class driver list, instead of the normal INF search. + DI_COMPAT_FROM_CLASS DI_FLAGS = 0x00080000 + + // This flag is set if the Class Install params should be used. + DI_CLASSINSTALLPARAMS DI_FLAGS = 0x00100000 + + // This flag is set if the caller of DiCallClassInstaller does NOT want the internal default action performed if the Class installer returns ERROR_DI_DO_DEFAULT. + DI_NODI_DEFAULTACTION DI_FLAGS = 0x00200000 + + // Flags for device installation + DI_QUIETINSTALL DI_FLAGS = 0x00800000 // don't confuse the user with questions or excess info + DI_NOFILECOPY DI_FLAGS = 0x01000000 // No file Copy necessary + DI_FORCECOPY DI_FLAGS = 0x02000000 // Force files to be copied from install path + DI_DRIVERPAGE_ADDED DI_FLAGS = 0x04000000 // Prop provider added Driver page. + DI_USECI_SELECTSTRINGS DI_FLAGS = 0x08000000 // Use Class Installer Provided strings in the Select Device Dlg + DI_OVERRIDE_INFFLAGS DI_FLAGS = 0x10000000 // Override INF flags + DI_PROPS_NOCHANGEUSAGE DI_FLAGS = 0x20000000 // No Enable/Disable in General Props + + DI_NOSELECTICONS DI_FLAGS = 0x40000000 // No small icons in select device dialogs + + DI_NOWRITE_IDS DI_FLAGS = 0x80000000 // Don't write HW & Compat IDs on install +) + +// DI_FLAGSEX is SP_DEVINSTALL_PARAMS.FlagsEx values +type DI_FLAGSEX uint32 + +const ( + DI_FLAGSEX_CI_FAILED DI_FLAGSEX = 0x00000004 // Failed to Load/Call class installer + DI_FLAGSEX_FINISHINSTALL_ACTION DI_FLAGSEX = 0x00000008 // Class/co-installer wants to get a DIF_FINISH_INSTALL action in client context. + DI_FLAGSEX_DIDINFOLIST DI_FLAGSEX = 0x00000010 // Did the Class Info List + DI_FLAGSEX_DIDCOMPATINFO DI_FLAGSEX = 0x00000020 // Did the Compat Info List + DI_FLAGSEX_FILTERCLASSES DI_FLAGSEX = 0x00000040 + DI_FLAGSEX_SETFAILEDINSTALL DI_FLAGSEX = 0x00000080 + DI_FLAGSEX_DEVICECHANGE DI_FLAGSEX = 0x00000100 + DI_FLAGSEX_ALWAYSWRITEIDS DI_FLAGSEX = 0x00000200 + DI_FLAGSEX_PROPCHANGE_PENDING DI_FLAGSEX = 0x00000400 // One or more device property sheets have had changes made to them, and need to have a DIF_PROPERTYCHANGE occur. + DI_FLAGSEX_ALLOWEXCLUDEDDRVS DI_FLAGSEX = 0x00000800 + DI_FLAGSEX_NOUIONQUERYREMOVE DI_FLAGSEX = 0x00001000 + DI_FLAGSEX_USECLASSFORCOMPAT DI_FLAGSEX = 0x00002000 // Use the device's class when building compat drv list. (Ignored if DI_COMPAT_FROM_CLASS flag is specified.) + DI_FLAGSEX_NO_DRVREG_MODIFY DI_FLAGSEX = 0x00008000 // Don't run AddReg and DelReg for device's software (driver) key. + DI_FLAGSEX_IN_SYSTEM_SETUP DI_FLAGSEX = 0x00010000 // Installation is occurring during initial system setup. + DI_FLAGSEX_INET_DRIVER DI_FLAGSEX = 0x00020000 // Driver came from Windows Update + DI_FLAGSEX_APPENDDRIVERLIST DI_FLAGSEX = 0x00040000 // Cause SetupDiBuildDriverInfoList to append a new driver list to an existing list. + DI_FLAGSEX_PREINSTALLBACKUP DI_FLAGSEX = 0x00080000 // not used + DI_FLAGSEX_BACKUPONREPLACE DI_FLAGSEX = 0x00100000 // not used + DI_FLAGSEX_DRIVERLIST_FROM_URL DI_FLAGSEX = 0x00200000 // build driver list from INF(s) retrieved from URL specified in SP_DEVINSTALL_PARAMS.DriverPath (empty string means Windows Update website) + DI_FLAGSEX_EXCLUDE_OLD_INET_DRIVERS DI_FLAGSEX = 0x00800000 // Don't include old Internet drivers when building a driver list. Ignored on Windows Vista and later. + DI_FLAGSEX_POWERPAGE_ADDED DI_FLAGSEX = 0x01000000 // class installer added their own power page + DI_FLAGSEX_FILTERSIMILARDRIVERS DI_FLAGSEX = 0x02000000 // only include similar drivers in class list + DI_FLAGSEX_INSTALLEDDRIVER DI_FLAGSEX = 0x04000000 // only add the installed driver to the class or compat driver list. Used in calls to SetupDiBuildDriverInfoList + DI_FLAGSEX_NO_CLASSLIST_NODE_MERGE DI_FLAGSEX = 0x08000000 // Don't remove identical driver nodes from the class list + DI_FLAGSEX_ALTPLATFORM_DRVSEARCH DI_FLAGSEX = 0x10000000 // Build driver list based on alternate platform information specified in associated file queue + DI_FLAGSEX_RESTART_DEVICE_ONLY DI_FLAGSEX = 0x20000000 // only restart the device drivers are being installed on as opposed to restarting all devices using those drivers. + DI_FLAGSEX_RECURSIVESEARCH DI_FLAGSEX = 0x40000000 // Tell SetupDiBuildDriverInfoList to do a recursive search + DI_FLAGSEX_SEARCH_PUBLISHED_INFS DI_FLAGSEX = 0x80000000 // Tell SetupDiBuildDriverInfoList to do a "published INF" search +) + +// ClassInstallHeader is the first member of any class install parameters structure. It contains the device installation request code that defines the format of the rest of the install parameters structure. +type ClassInstallHeader struct { + size uint32 + InstallFunction DI_FUNCTION +} + +func MakeClassInstallHeader(installFunction DI_FUNCTION) *ClassInstallHeader { + hdr := &ClassInstallHeader{InstallFunction: installFunction} + hdr.size = uint32(unsafe.Sizeof(*hdr)) + return hdr +} + +// DICS_STATE specifies values indicating a change in a device's state +type DICS_STATE uint32 + +const ( + DICS_ENABLE DICS_STATE = 0x00000001 // The device is being enabled. + DICS_DISABLE DICS_STATE = 0x00000002 // The device is being disabled. + DICS_PROPCHANGE DICS_STATE = 0x00000003 // The properties of the device have changed. + DICS_START DICS_STATE = 0x00000004 // The device is being started (if the request is for the currently active hardware profile). + DICS_STOP DICS_STATE = 0x00000005 // The device is being stopped. The driver stack will be unloaded and the CSCONFIGFLAG_DO_NOT_START flag will be set for the device. +) + +// DICS_FLAG specifies the scope of a device property change +type DICS_FLAG uint32 + +const ( + DICS_FLAG_GLOBAL DICS_FLAG = 0x00000001 // make change in all hardware profiles + DICS_FLAG_CONFIGSPECIFIC DICS_FLAG = 0x00000002 // make change in specified profile only + DICS_FLAG_CONFIGGENERAL DICS_FLAG = 0x00000004 // 1 or more hardware profile-specific changes to follow (obsolete) +) + +// PropChangeParams is a structure corresponding to a DIF_PROPERTYCHANGE install function. +type PropChangeParams struct { + ClassInstallHeader ClassInstallHeader + StateChange DICS_STATE + Scope DICS_FLAG + HwProfile uint32 +} + +// DI_REMOVEDEVICE specifies the scope of the device removal +type DI_REMOVEDEVICE uint32 + +const ( + DI_REMOVEDEVICE_GLOBAL DI_REMOVEDEVICE = 0x00000001 // Make this change in all hardware profiles. Remove information about the device from the registry. + DI_REMOVEDEVICE_CONFIGSPECIFIC DI_REMOVEDEVICE = 0x00000002 // Make this change to only the hardware profile specified by HwProfile. this flag only applies to root-enumerated devices. When Windows removes the device from the last hardware profile in which it was configured, Windows performs a global removal. +) + +// RemoveDeviceParams is a structure corresponding to a DIF_REMOVE install function. +type RemoveDeviceParams struct { + ClassInstallHeader ClassInstallHeader + Scope DI_REMOVEDEVICE + HwProfile uint32 +} + +// DrvInfoData is driver information structure (member of a driver info list that may be associated with a particular device instance, or (globally) with a device information set) +type DrvInfoData struct { + size uint32 + DriverType uint32 + _ uintptr + description [LINE_LEN]uint16 + mfgName [LINE_LEN]uint16 + providerName [LINE_LEN]uint16 + DriverDate Filetime + DriverVersion uint64 +} + +func (data *DrvInfoData) Description() string { + return UTF16ToString(data.description[:]) +} + +func (data *DrvInfoData) SetDescription(description string) error { + str, err := UTF16FromString(description) + if err != nil { + return err + } + copy(data.description[:], str) + return nil +} + +func (data *DrvInfoData) MfgName() string { + return UTF16ToString(data.mfgName[:]) +} + +func (data *DrvInfoData) SetMfgName(mfgName string) error { + str, err := UTF16FromString(mfgName) + if err != nil { + return err + } + copy(data.mfgName[:], str) + return nil +} + +func (data *DrvInfoData) ProviderName() string { + return UTF16ToString(data.providerName[:]) +} + +func (data *DrvInfoData) SetProviderName(providerName string) error { + str, err := UTF16FromString(providerName) + if err != nil { + return err + } + copy(data.providerName[:], str) + return nil +} + +// IsNewer method returns true if DrvInfoData date and version is newer than supplied parameters. +func (data *DrvInfoData) IsNewer(driverDate Filetime, driverVersion uint64) bool { + if data.DriverDate.HighDateTime > driverDate.HighDateTime { + return true + } + if data.DriverDate.HighDateTime < driverDate.HighDateTime { + return false + } + + if data.DriverDate.LowDateTime > driverDate.LowDateTime { + return true + } + if data.DriverDate.LowDateTime < driverDate.LowDateTime { + return false + } + + if data.DriverVersion > driverVersion { + return true + } + if data.DriverVersion < driverVersion { + return false + } + + return false +} + +// DrvInfoDetailData is driver information details structure (provides detailed information about a particular driver information structure) +type DrvInfoDetailData struct { + size uint32 // Use unsafeSizeOf method + InfDate Filetime + compatIDsOffset uint32 + compatIDsLength uint32 + _ uintptr + sectionName [LINE_LEN]uint16 + infFileName [MAX_PATH]uint16 + drvDescription [LINE_LEN]uint16 + hardwareID [1]uint16 +} + +func (*DrvInfoDetailData) unsafeSizeOf() uint32 { + if unsafe.Sizeof(uintptr(0)) == 4 { + // Windows declares this with pshpack1.h + return uint32(unsafe.Offsetof(DrvInfoDetailData{}.hardwareID) + unsafe.Sizeof(DrvInfoDetailData{}.hardwareID)) + } + return uint32(unsafe.Sizeof(DrvInfoDetailData{})) +} + +func (data *DrvInfoDetailData) SectionName() string { + return UTF16ToString(data.sectionName[:]) +} + +func (data *DrvInfoDetailData) InfFileName() string { + return UTF16ToString(data.infFileName[:]) +} + +func (data *DrvInfoDetailData) DrvDescription() string { + return UTF16ToString(data.drvDescription[:]) +} + +func (data *DrvInfoDetailData) HardwareID() string { + if data.compatIDsOffset > 1 { + bufW := data.getBuf() + return UTF16ToString(bufW[:wcslen(bufW)]) + } + + return "" +} + +func (data *DrvInfoDetailData) CompatIDs() []string { + a := make([]string, 0) + + if data.compatIDsLength > 0 { + bufW := data.getBuf() + bufW = bufW[data.compatIDsOffset : data.compatIDsOffset+data.compatIDsLength] + for i := 0; i < len(bufW); { + j := i + wcslen(bufW[i:]) + if i < j { + a = append(a, UTF16ToString(bufW[i:j])) + } + i = j + 1 + } + } + + return a +} + +func (data *DrvInfoDetailData) getBuf() []uint16 { + len := (data.size - uint32(unsafe.Offsetof(data.hardwareID))) / 2 + sl := struct { + addr *uint16 + len int + cap int + }{&data.hardwareID[0], int(len), int(len)} + return *(*[]uint16)(unsafe.Pointer(&sl)) +} + +// IsCompatible method tests if given hardware ID matches the driver or is listed on the compatible ID list. +func (data *DrvInfoDetailData) IsCompatible(hwid string) bool { + hwidLC := strings.ToLower(hwid) + if strings.ToLower(data.HardwareID()) == hwidLC { + return true + } + a := data.CompatIDs() + for i := range a { + if strings.ToLower(a[i]) == hwidLC { + return true + } + } + + return false +} + +// DICD flags control SetupDiCreateDeviceInfo +type DICD uint32 + +const ( + DICD_GENERATE_ID DICD = 0x00000001 + DICD_INHERIT_CLASSDRVS DICD = 0x00000002 +) + +// SUOI flags control SetupUninstallOEMInf +type SUOI uint32 + +const ( + SUOI_FORCEDELETE SUOI = 0x0001 +) + +// SPDIT flags to distinguish between class drivers and +// device drivers. (Passed in 'DriverType' parameter of +// driver information list APIs) +type SPDIT uint32 + +const ( + SPDIT_NODRIVER SPDIT = 0x00000000 + SPDIT_CLASSDRIVER SPDIT = 0x00000001 + SPDIT_COMPATDRIVER SPDIT = 0x00000002 +) + +// DIGCF flags control what is included in the device information set built by SetupDiGetClassDevs +type DIGCF uint32 + +const ( + DIGCF_DEFAULT DIGCF = 0x00000001 // only valid with DIGCF_DEVICEINTERFACE + DIGCF_PRESENT DIGCF = 0x00000002 + DIGCF_ALLCLASSES DIGCF = 0x00000004 + DIGCF_PROFILE DIGCF = 0x00000008 + DIGCF_DEVICEINTERFACE DIGCF = 0x00000010 +) + +// DIREG specifies values for SetupDiCreateDevRegKey, SetupDiOpenDevRegKey, and SetupDiDeleteDevRegKey. +type DIREG uint32 + +const ( + DIREG_DEV DIREG = 0x00000001 // Open/Create/Delete device key + DIREG_DRV DIREG = 0x00000002 // Open/Create/Delete driver key + DIREG_BOTH DIREG = 0x00000004 // Delete both driver and Device key +) + +// SPDRP specifies device registry property codes +// (Codes marked as read-only (R) may only be used for +// SetupDiGetDeviceRegistryProperty) +// +// These values should cover the same set of registry properties +// as defined by the CM_DRP codes in cfgmgr32.h. +// +// Note that SPDRP codes are zero based while CM_DRP codes are one based! +type SPDRP uint32 + +const ( + SPDRP_DEVICEDESC SPDRP = 0x00000000 // DeviceDesc (R/W) + SPDRP_HARDWAREID SPDRP = 0x00000001 // HardwareID (R/W) + SPDRP_COMPATIBLEIDS SPDRP = 0x00000002 // CompatibleIDs (R/W) + SPDRP_SERVICE SPDRP = 0x00000004 // Service (R/W) + SPDRP_CLASS SPDRP = 0x00000007 // Class (R--tied to ClassGUID) + SPDRP_CLASSGUID SPDRP = 0x00000008 // ClassGUID (R/W) + SPDRP_DRIVER SPDRP = 0x00000009 // Driver (R/W) + SPDRP_CONFIGFLAGS SPDRP = 0x0000000A // ConfigFlags (R/W) + SPDRP_MFG SPDRP = 0x0000000B // Mfg (R/W) + SPDRP_FRIENDLYNAME SPDRP = 0x0000000C // FriendlyName (R/W) + SPDRP_LOCATION_INFORMATION SPDRP = 0x0000000D // LocationInformation (R/W) + SPDRP_PHYSICAL_DEVICE_OBJECT_NAME SPDRP = 0x0000000E // PhysicalDeviceObjectName (R) + SPDRP_CAPABILITIES SPDRP = 0x0000000F // Capabilities (R) + SPDRP_UI_NUMBER SPDRP = 0x00000010 // UiNumber (R) + SPDRP_UPPERFILTERS SPDRP = 0x00000011 // UpperFilters (R/W) + SPDRP_LOWERFILTERS SPDRP = 0x00000012 // LowerFilters (R/W) + SPDRP_BUSTYPEGUID SPDRP = 0x00000013 // BusTypeGUID (R) + SPDRP_LEGACYBUSTYPE SPDRP = 0x00000014 // LegacyBusType (R) + SPDRP_BUSNUMBER SPDRP = 0x00000015 // BusNumber (R) + SPDRP_ENUMERATOR_NAME SPDRP = 0x00000016 // Enumerator Name (R) + SPDRP_SECURITY SPDRP = 0x00000017 // Security (R/W, binary form) + SPDRP_SECURITY_SDS SPDRP = 0x00000018 // Security (W, SDS form) + SPDRP_DEVTYPE SPDRP = 0x00000019 // Device Type (R/W) + SPDRP_EXCLUSIVE SPDRP = 0x0000001A // Device is exclusive-access (R/W) + SPDRP_CHARACTERISTICS SPDRP = 0x0000001B // Device Characteristics (R/W) + SPDRP_ADDRESS SPDRP = 0x0000001C // Device Address (R) + SPDRP_UI_NUMBER_DESC_FORMAT SPDRP = 0x0000001D // UiNumberDescFormat (R/W) + SPDRP_DEVICE_POWER_DATA SPDRP = 0x0000001E // Device Power Data (R) + SPDRP_REMOVAL_POLICY SPDRP = 0x0000001F // Removal Policy (R) + SPDRP_REMOVAL_POLICY_HW_DEFAULT SPDRP = 0x00000020 // Hardware Removal Policy (R) + SPDRP_REMOVAL_POLICY_OVERRIDE SPDRP = 0x00000021 // Removal Policy Override (RW) + SPDRP_INSTALL_STATE SPDRP = 0x00000022 // Device Install State (R) + SPDRP_LOCATION_PATHS SPDRP = 0x00000023 // Device Location Paths (R) + SPDRP_BASE_CONTAINERID SPDRP = 0x00000024 // Base ContainerID (R) + + SPDRP_MAXIMUM_PROPERTY SPDRP = 0x00000025 // Upper bound on ordinals +) + +// DEVPROPTYPE represents the property-data-type identifier that specifies the +// data type of a device property value in the unified device property model. +type DEVPROPTYPE uint32 + +const ( + DEVPROP_TYPEMOD_ARRAY DEVPROPTYPE = 0x00001000 + DEVPROP_TYPEMOD_LIST DEVPROPTYPE = 0x00002000 + + DEVPROP_TYPE_EMPTY DEVPROPTYPE = 0x00000000 + DEVPROP_TYPE_NULL DEVPROPTYPE = 0x00000001 + DEVPROP_TYPE_SBYTE DEVPROPTYPE = 0x00000002 + DEVPROP_TYPE_BYTE DEVPROPTYPE = 0x00000003 + DEVPROP_TYPE_INT16 DEVPROPTYPE = 0x00000004 + DEVPROP_TYPE_UINT16 DEVPROPTYPE = 0x00000005 + DEVPROP_TYPE_INT32 DEVPROPTYPE = 0x00000006 + DEVPROP_TYPE_UINT32 DEVPROPTYPE = 0x00000007 + DEVPROP_TYPE_INT64 DEVPROPTYPE = 0x00000008 + DEVPROP_TYPE_UINT64 DEVPROPTYPE = 0x00000009 + DEVPROP_TYPE_FLOAT DEVPROPTYPE = 0x0000000A + DEVPROP_TYPE_DOUBLE DEVPROPTYPE = 0x0000000B + DEVPROP_TYPE_DECIMAL DEVPROPTYPE = 0x0000000C + DEVPROP_TYPE_GUID DEVPROPTYPE = 0x0000000D + DEVPROP_TYPE_CURRENCY DEVPROPTYPE = 0x0000000E + DEVPROP_TYPE_DATE DEVPROPTYPE = 0x0000000F + DEVPROP_TYPE_FILETIME DEVPROPTYPE = 0x00000010 + DEVPROP_TYPE_BOOLEAN DEVPROPTYPE = 0x00000011 + DEVPROP_TYPE_STRING DEVPROPTYPE = 0x00000012 + DEVPROP_TYPE_STRING_LIST DEVPROPTYPE = DEVPROP_TYPE_STRING | DEVPROP_TYPEMOD_LIST + DEVPROP_TYPE_SECURITY_DESCRIPTOR DEVPROPTYPE = 0x00000013 + DEVPROP_TYPE_SECURITY_DESCRIPTOR_STRING DEVPROPTYPE = 0x00000014 + DEVPROP_TYPE_DEVPROPKEY DEVPROPTYPE = 0x00000015 + DEVPROP_TYPE_DEVPROPTYPE DEVPROPTYPE = 0x00000016 + DEVPROP_TYPE_BINARY DEVPROPTYPE = DEVPROP_TYPE_BYTE | DEVPROP_TYPEMOD_ARRAY + DEVPROP_TYPE_ERROR DEVPROPTYPE = 0x00000017 + DEVPROP_TYPE_NTSTATUS DEVPROPTYPE = 0x00000018 + DEVPROP_TYPE_STRING_INDIRECT DEVPROPTYPE = 0x00000019 + + MAX_DEVPROP_TYPE DEVPROPTYPE = 0x00000019 + MAX_DEVPROP_TYPEMOD DEVPROPTYPE = 0x00002000 + + DEVPROP_MASK_TYPE DEVPROPTYPE = 0x00000FFF + DEVPROP_MASK_TYPEMOD DEVPROPTYPE = 0x0000F000 +) + +// DEVPROPGUID specifies a property category. +type DEVPROPGUID GUID + +// DEVPROPID uniquely identifies the property within the property category. +type DEVPROPID uint32 + +const DEVPROPID_FIRST_USABLE DEVPROPID = 2 + +// DEVPROPKEY represents a device property key for a device property in the +// unified device property model. +type DEVPROPKEY struct { + FmtID DEVPROPGUID + PID DEVPROPID +} + +// CONFIGRET is a return value or error code from cfgmgr32 APIs +type CONFIGRET uint32 + +func (ret CONFIGRET) Error() string { + if win32Error, ok := ret.Unwrap().(Errno); ok { + return fmt.Sprintf("%s (CfgMgr error: 0x%08x)", win32Error.Error(), uint32(ret)) + } + return fmt.Sprintf("CfgMgr error: 0x%08x", uint32(ret)) +} + +func (ret CONFIGRET) Win32Error(defaultError Errno) Errno { + return cm_MapCrToWin32Err(ret, defaultError) +} + +func (ret CONFIGRET) Unwrap() error { + const noMatch = Errno(^uintptr(0)) + win32Error := ret.Win32Error(noMatch) + if win32Error == noMatch { + return nil + } + return win32Error +} + +const ( + CR_SUCCESS CONFIGRET = 0x00000000 + CR_DEFAULT CONFIGRET = 0x00000001 + CR_OUT_OF_MEMORY CONFIGRET = 0x00000002 + CR_INVALID_POINTER CONFIGRET = 0x00000003 + CR_INVALID_FLAG CONFIGRET = 0x00000004 + CR_INVALID_DEVNODE CONFIGRET = 0x00000005 + CR_INVALID_DEVINST = CR_INVALID_DEVNODE + CR_INVALID_RES_DES CONFIGRET = 0x00000006 + CR_INVALID_LOG_CONF CONFIGRET = 0x00000007 + CR_INVALID_ARBITRATOR CONFIGRET = 0x00000008 + CR_INVALID_NODELIST CONFIGRET = 0x00000009 + CR_DEVNODE_HAS_REQS CONFIGRET = 0x0000000A + CR_DEVINST_HAS_REQS = CR_DEVNODE_HAS_REQS + CR_INVALID_RESOURCEID CONFIGRET = 0x0000000B + CR_DLVXD_NOT_FOUND CONFIGRET = 0x0000000C + CR_NO_SUCH_DEVNODE CONFIGRET = 0x0000000D + CR_NO_SUCH_DEVINST = CR_NO_SUCH_DEVNODE + CR_NO_MORE_LOG_CONF CONFIGRET = 0x0000000E + CR_NO_MORE_RES_DES CONFIGRET = 0x0000000F + CR_ALREADY_SUCH_DEVNODE CONFIGRET = 0x00000010 + CR_ALREADY_SUCH_DEVINST = CR_ALREADY_SUCH_DEVNODE + CR_INVALID_RANGE_LIST CONFIGRET = 0x00000011 + CR_INVALID_RANGE CONFIGRET = 0x00000012 + CR_FAILURE CONFIGRET = 0x00000013 + CR_NO_SUCH_LOGICAL_DEV CONFIGRET = 0x00000014 + CR_CREATE_BLOCKED CONFIGRET = 0x00000015 + CR_NOT_SYSTEM_VM CONFIGRET = 0x00000016 + CR_REMOVE_VETOED CONFIGRET = 0x00000017 + CR_APM_VETOED CONFIGRET = 0x00000018 + CR_INVALID_LOAD_TYPE CONFIGRET = 0x00000019 + CR_BUFFER_SMALL CONFIGRET = 0x0000001A + CR_NO_ARBITRATOR CONFIGRET = 0x0000001B + CR_NO_REGISTRY_HANDLE CONFIGRET = 0x0000001C + CR_REGISTRY_ERROR CONFIGRET = 0x0000001D + CR_INVALID_DEVICE_ID CONFIGRET = 0x0000001E + CR_INVALID_DATA CONFIGRET = 0x0000001F + CR_INVALID_API CONFIGRET = 0x00000020 + CR_DEVLOADER_NOT_READY CONFIGRET = 0x00000021 + CR_NEED_RESTART CONFIGRET = 0x00000022 + CR_NO_MORE_HW_PROFILES CONFIGRET = 0x00000023 + CR_DEVICE_NOT_THERE CONFIGRET = 0x00000024 + CR_NO_SUCH_VALUE CONFIGRET = 0x00000025 + CR_WRONG_TYPE CONFIGRET = 0x00000026 + CR_INVALID_PRIORITY CONFIGRET = 0x00000027 + CR_NOT_DISABLEABLE CONFIGRET = 0x00000028 + CR_FREE_RESOURCES CONFIGRET = 0x00000029 + CR_QUERY_VETOED CONFIGRET = 0x0000002A + CR_CANT_SHARE_IRQ CONFIGRET = 0x0000002B + CR_NO_DEPENDENT CONFIGRET = 0x0000002C + CR_SAME_RESOURCES CONFIGRET = 0x0000002D + CR_NO_SUCH_REGISTRY_KEY CONFIGRET = 0x0000002E + CR_INVALID_MACHINENAME CONFIGRET = 0x0000002F + CR_REMOTE_COMM_FAILURE CONFIGRET = 0x00000030 + CR_MACHINE_UNAVAILABLE CONFIGRET = 0x00000031 + CR_NO_CM_SERVICES CONFIGRET = 0x00000032 + CR_ACCESS_DENIED CONFIGRET = 0x00000033 + CR_CALL_NOT_IMPLEMENTED CONFIGRET = 0x00000034 + CR_INVALID_PROPERTY CONFIGRET = 0x00000035 + CR_DEVICE_INTERFACE_ACTIVE CONFIGRET = 0x00000036 + CR_NO_SUCH_DEVICE_INTERFACE CONFIGRET = 0x00000037 + CR_INVALID_REFERENCE_STRING CONFIGRET = 0x00000038 + CR_INVALID_CONFLICT_LIST CONFIGRET = 0x00000039 + CR_INVALID_INDEX CONFIGRET = 0x0000003A + CR_INVALID_STRUCTURE_SIZE CONFIGRET = 0x0000003B + NUM_CR_RESULTS CONFIGRET = 0x0000003C +) + +const ( + CM_GET_DEVICE_INTERFACE_LIST_PRESENT = 0 // only currently 'live' device interfaces + CM_GET_DEVICE_INTERFACE_LIST_ALL_DEVICES = 1 // all registered device interfaces, live or not +) + +const ( + DN_ROOT_ENUMERATED = 0x00000001 // Was enumerated by ROOT + DN_DRIVER_LOADED = 0x00000002 // Has Register_Device_Driver + DN_ENUM_LOADED = 0x00000004 // Has Register_Enumerator + DN_STARTED = 0x00000008 // Is currently configured + DN_MANUAL = 0x00000010 // Manually installed + DN_NEED_TO_ENUM = 0x00000020 // May need reenumeration + DN_NOT_FIRST_TIME = 0x00000040 // Has received a config + DN_HARDWARE_ENUM = 0x00000080 // Enum generates hardware ID + DN_LIAR = 0x00000100 // Lied about can reconfig once + DN_HAS_MARK = 0x00000200 // Not CM_Create_DevInst lately + DN_HAS_PROBLEM = 0x00000400 // Need device installer + DN_FILTERED = 0x00000800 // Is filtered + DN_MOVED = 0x00001000 // Has been moved + DN_DISABLEABLE = 0x00002000 // Can be disabled + DN_REMOVABLE = 0x00004000 // Can be removed + DN_PRIVATE_PROBLEM = 0x00008000 // Has a private problem + DN_MF_PARENT = 0x00010000 // Multi function parent + DN_MF_CHILD = 0x00020000 // Multi function child + DN_WILL_BE_REMOVED = 0x00040000 // DevInst is being removed + DN_NOT_FIRST_TIMEE = 0x00080000 // Has received a config enumerate + DN_STOP_FREE_RES = 0x00100000 // When child is stopped, free resources + DN_REBAL_CANDIDATE = 0x00200000 // Don't skip during rebalance + DN_BAD_PARTIAL = 0x00400000 // This devnode's log_confs do not have same resources + DN_NT_ENUMERATOR = 0x00800000 // This devnode's is an NT enumerator + DN_NT_DRIVER = 0x01000000 // This devnode's is an NT driver + DN_NEEDS_LOCKING = 0x02000000 // Devnode need lock resume processing + DN_ARM_WAKEUP = 0x04000000 // Devnode can be the wakeup device + DN_APM_ENUMERATOR = 0x08000000 // APM aware enumerator + DN_APM_DRIVER = 0x10000000 // APM aware driver + DN_SILENT_INSTALL = 0x20000000 // Silent install + DN_NO_SHOW_IN_DM = 0x40000000 // No show in device manager + DN_BOOT_LOG_PROB = 0x80000000 // Had a problem during preassignment of boot log conf + DN_NEED_RESTART = DN_LIAR // System needs to be restarted for this Devnode to work properly + DN_DRIVER_BLOCKED = DN_NOT_FIRST_TIME // One or more drivers are blocked from loading for this Devnode + DN_LEGACY_DRIVER = DN_MOVED // This device is using a legacy driver + DN_CHILD_WITH_INVALID_ID = DN_HAS_MARK // One or more children have invalid IDs + DN_DEVICE_DISCONNECTED = DN_NEEDS_LOCKING // The function driver for a device reported that the device is not connected. Typically this means a wireless device is out of range. + DN_QUERY_REMOVE_PENDING = DN_MF_PARENT // Device is part of a set of related devices collectively pending query-removal + DN_QUERY_REMOVE_ACTIVE = DN_MF_CHILD // Device is actively engaged in a query-remove IRP + DN_CHANGEABLE_FLAGS = DN_NOT_FIRST_TIME | DN_HARDWARE_ENUM | DN_HAS_MARK | DN_DISABLEABLE | DN_REMOVABLE | DN_MF_CHILD | DN_MF_PARENT | DN_NOT_FIRST_TIMEE | DN_STOP_FREE_RES | DN_REBAL_CANDIDATE | DN_NT_ENUMERATOR | DN_NT_DRIVER | DN_SILENT_INSTALL | DN_NO_SHOW_IN_DM +) + +//sys setupDiCreateDeviceInfoListEx(classGUID *GUID, hwndParent uintptr, machineName *uint16, reserved uintptr) (handle DevInfo, err error) [failretval==DevInfo(InvalidHandle)] = setupapi.SetupDiCreateDeviceInfoListExW + +// SetupDiCreateDeviceInfoListEx function creates an empty device information set on a remote or a local computer and optionally associates the set with a device setup class. +func SetupDiCreateDeviceInfoListEx(classGUID *GUID, hwndParent uintptr, machineName string) (deviceInfoSet DevInfo, err error) { + var machineNameUTF16 *uint16 + if machineName != "" { + machineNameUTF16, err = UTF16PtrFromString(machineName) + if err != nil { + return + } + } + return setupDiCreateDeviceInfoListEx(classGUID, hwndParent, machineNameUTF16, 0) +} + +//sys setupDiGetDeviceInfoListDetail(deviceInfoSet DevInfo, deviceInfoSetDetailData *DevInfoListDetailData) (err error) = setupapi.SetupDiGetDeviceInfoListDetailW + +// SetupDiGetDeviceInfoListDetail function retrieves information associated with a device information set including the class GUID, remote computer handle, and remote computer name. +func SetupDiGetDeviceInfoListDetail(deviceInfoSet DevInfo) (deviceInfoSetDetailData *DevInfoListDetailData, err error) { + data := &DevInfoListDetailData{} + data.size = data.unsafeSizeOf() + + return data, setupDiGetDeviceInfoListDetail(deviceInfoSet, data) +} + +// DeviceInfoListDetail method retrieves information associated with a device information set including the class GUID, remote computer handle, and remote computer name. +func (deviceInfoSet DevInfo) DeviceInfoListDetail() (*DevInfoListDetailData, error) { + return SetupDiGetDeviceInfoListDetail(deviceInfoSet) +} + +//sys setupDiCreateDeviceInfo(deviceInfoSet DevInfo, DeviceName *uint16, classGUID *GUID, DeviceDescription *uint16, hwndParent uintptr, CreationFlags DICD, deviceInfoData *DevInfoData) (err error) = setupapi.SetupDiCreateDeviceInfoW + +// SetupDiCreateDeviceInfo function creates a new device information element and adds it as a new member to the specified device information set. +func SetupDiCreateDeviceInfo(deviceInfoSet DevInfo, deviceName string, classGUID *GUID, deviceDescription string, hwndParent uintptr, creationFlags DICD) (deviceInfoData *DevInfoData, err error) { + deviceNameUTF16, err := UTF16PtrFromString(deviceName) + if err != nil { + return + } + + var deviceDescriptionUTF16 *uint16 + if deviceDescription != "" { + deviceDescriptionUTF16, err = UTF16PtrFromString(deviceDescription) + if err != nil { + return + } + } + + data := &DevInfoData{} + data.size = uint32(unsafe.Sizeof(*data)) + + return data, setupDiCreateDeviceInfo(deviceInfoSet, deviceNameUTF16, classGUID, deviceDescriptionUTF16, hwndParent, creationFlags, data) +} + +// CreateDeviceInfo method creates a new device information element and adds it as a new member to the specified device information set. +func (deviceInfoSet DevInfo) CreateDeviceInfo(deviceName string, classGUID *GUID, deviceDescription string, hwndParent uintptr, creationFlags DICD) (*DevInfoData, error) { + return SetupDiCreateDeviceInfo(deviceInfoSet, deviceName, classGUID, deviceDescription, hwndParent, creationFlags) +} + +//sys setupDiEnumDeviceInfo(deviceInfoSet DevInfo, memberIndex uint32, deviceInfoData *DevInfoData) (err error) = setupapi.SetupDiEnumDeviceInfo + +// SetupDiEnumDeviceInfo function returns a DevInfoData structure that specifies a device information element in a device information set. +func SetupDiEnumDeviceInfo(deviceInfoSet DevInfo, memberIndex int) (*DevInfoData, error) { + data := &DevInfoData{} + data.size = uint32(unsafe.Sizeof(*data)) + + return data, setupDiEnumDeviceInfo(deviceInfoSet, uint32(memberIndex), data) +} + +// EnumDeviceInfo method returns a DevInfoData structure that specifies a device information element in a device information set. +func (deviceInfoSet DevInfo) EnumDeviceInfo(memberIndex int) (*DevInfoData, error) { + return SetupDiEnumDeviceInfo(deviceInfoSet, memberIndex) +} + +// SetupDiDestroyDeviceInfoList function deletes a device information set and frees all associated memory. +//sys SetupDiDestroyDeviceInfoList(deviceInfoSet DevInfo) (err error) = setupapi.SetupDiDestroyDeviceInfoList + +// Close method deletes a device information set and frees all associated memory. +func (deviceInfoSet DevInfo) Close() error { + return SetupDiDestroyDeviceInfoList(deviceInfoSet) +} + +//sys SetupDiBuildDriverInfoList(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverType SPDIT) (err error) = setupapi.SetupDiBuildDriverInfoList + +// BuildDriverInfoList method builds a list of drivers that is associated with a specific device or with the global class driver list for a device information set. +func (deviceInfoSet DevInfo) BuildDriverInfoList(deviceInfoData *DevInfoData, driverType SPDIT) error { + return SetupDiBuildDriverInfoList(deviceInfoSet, deviceInfoData, driverType) +} + +//sys SetupDiCancelDriverInfoSearch(deviceInfoSet DevInfo) (err error) = setupapi.SetupDiCancelDriverInfoSearch + +// CancelDriverInfoSearch method cancels a driver list search that is currently in progress in a different thread. +func (deviceInfoSet DevInfo) CancelDriverInfoSearch() error { + return SetupDiCancelDriverInfoSearch(deviceInfoSet) +} + +//sys setupDiEnumDriverInfo(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverType SPDIT, memberIndex uint32, driverInfoData *DrvInfoData) (err error) = setupapi.SetupDiEnumDriverInfoW + +// SetupDiEnumDriverInfo function enumerates the members of a driver list. +func SetupDiEnumDriverInfo(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverType SPDIT, memberIndex int) (*DrvInfoData, error) { + data := &DrvInfoData{} + data.size = uint32(unsafe.Sizeof(*data)) + + return data, setupDiEnumDriverInfo(deviceInfoSet, deviceInfoData, driverType, uint32(memberIndex), data) +} + +// EnumDriverInfo method enumerates the members of a driver list. +func (deviceInfoSet DevInfo) EnumDriverInfo(deviceInfoData *DevInfoData, driverType SPDIT, memberIndex int) (*DrvInfoData, error) { + return SetupDiEnumDriverInfo(deviceInfoSet, deviceInfoData, driverType, memberIndex) +} + +//sys setupDiGetSelectedDriver(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverInfoData *DrvInfoData) (err error) = setupapi.SetupDiGetSelectedDriverW + +// SetupDiGetSelectedDriver function retrieves the selected driver for a device information set or a particular device information element. +func SetupDiGetSelectedDriver(deviceInfoSet DevInfo, deviceInfoData *DevInfoData) (*DrvInfoData, error) { + data := &DrvInfoData{} + data.size = uint32(unsafe.Sizeof(*data)) + + return data, setupDiGetSelectedDriver(deviceInfoSet, deviceInfoData, data) +} + +// SelectedDriver method retrieves the selected driver for a device information set or a particular device information element. +func (deviceInfoSet DevInfo) SelectedDriver(deviceInfoData *DevInfoData) (*DrvInfoData, error) { + return SetupDiGetSelectedDriver(deviceInfoSet, deviceInfoData) +} + +//sys SetupDiSetSelectedDriver(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverInfoData *DrvInfoData) (err error) = setupapi.SetupDiSetSelectedDriverW + +// SetSelectedDriver method sets, or resets, the selected driver for a device information element or the selected class driver for a device information set. +func (deviceInfoSet DevInfo) SetSelectedDriver(deviceInfoData *DevInfoData, driverInfoData *DrvInfoData) error { + return SetupDiSetSelectedDriver(deviceInfoSet, deviceInfoData, driverInfoData) +} + +//sys setupDiGetDriverInfoDetail(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverInfoData *DrvInfoData, driverInfoDetailData *DrvInfoDetailData, driverInfoDetailDataSize uint32, requiredSize *uint32) (err error) = setupapi.SetupDiGetDriverInfoDetailW + +// SetupDiGetDriverInfoDetail function retrieves driver information detail for a device information set or a particular device information element in the device information set. +func SetupDiGetDriverInfoDetail(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverInfoData *DrvInfoData) (*DrvInfoDetailData, error) { + reqSize := uint32(2048) + for { + buf := make([]byte, reqSize) + data := (*DrvInfoDetailData)(unsafe.Pointer(&buf[0])) + data.size = data.unsafeSizeOf() + err := setupDiGetDriverInfoDetail(deviceInfoSet, deviceInfoData, driverInfoData, data, uint32(len(buf)), &reqSize) + if err == ERROR_INSUFFICIENT_BUFFER { + continue + } + if err != nil { + return nil, err + } + data.size = reqSize + return data, nil + } +} + +// DriverInfoDetail method retrieves driver information detail for a device information set or a particular device information element in the device information set. +func (deviceInfoSet DevInfo) DriverInfoDetail(deviceInfoData *DevInfoData, driverInfoData *DrvInfoData) (*DrvInfoDetailData, error) { + return SetupDiGetDriverInfoDetail(deviceInfoSet, deviceInfoData, driverInfoData) +} + +//sys SetupDiDestroyDriverInfoList(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverType SPDIT) (err error) = setupapi.SetupDiDestroyDriverInfoList + +// DestroyDriverInfoList method deletes a driver list. +func (deviceInfoSet DevInfo) DestroyDriverInfoList(deviceInfoData *DevInfoData, driverType SPDIT) error { + return SetupDiDestroyDriverInfoList(deviceInfoSet, deviceInfoData, driverType) +} + +//sys setupDiGetClassDevsEx(classGUID *GUID, Enumerator *uint16, hwndParent uintptr, Flags DIGCF, deviceInfoSet DevInfo, machineName *uint16, reserved uintptr) (handle DevInfo, err error) [failretval==DevInfo(InvalidHandle)] = setupapi.SetupDiGetClassDevsExW + +// SetupDiGetClassDevsEx function returns a handle to a device information set that contains requested device information elements for a local or a remote computer. +func SetupDiGetClassDevsEx(classGUID *GUID, enumerator string, hwndParent uintptr, flags DIGCF, deviceInfoSet DevInfo, machineName string) (handle DevInfo, err error) { + var enumeratorUTF16 *uint16 + if enumerator != "" { + enumeratorUTF16, err = UTF16PtrFromString(enumerator) + if err != nil { + return + } + } + var machineNameUTF16 *uint16 + if machineName != "" { + machineNameUTF16, err = UTF16PtrFromString(machineName) + if err != nil { + return + } + } + return setupDiGetClassDevsEx(classGUID, enumeratorUTF16, hwndParent, flags, deviceInfoSet, machineNameUTF16, 0) +} + +// SetupDiCallClassInstaller function calls the appropriate class installer, and any registered co-installers, with the specified installation request (DIF code). +//sys SetupDiCallClassInstaller(installFunction DI_FUNCTION, deviceInfoSet DevInfo, deviceInfoData *DevInfoData) (err error) = setupapi.SetupDiCallClassInstaller + +// CallClassInstaller member calls the appropriate class installer, and any registered co-installers, with the specified installation request (DIF code). +func (deviceInfoSet DevInfo) CallClassInstaller(installFunction DI_FUNCTION, deviceInfoData *DevInfoData) error { + return SetupDiCallClassInstaller(installFunction, deviceInfoSet, deviceInfoData) +} + +// SetupDiOpenDevRegKey function opens a registry key for device-specific configuration information. +//sys SetupDiOpenDevRegKey(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, Scope DICS_FLAG, HwProfile uint32, KeyType DIREG, samDesired uint32) (key Handle, err error) [failretval==InvalidHandle] = setupapi.SetupDiOpenDevRegKey + +// OpenDevRegKey method opens a registry key for device-specific configuration information. +func (deviceInfoSet DevInfo) OpenDevRegKey(DeviceInfoData *DevInfoData, Scope DICS_FLAG, HwProfile uint32, KeyType DIREG, samDesired uint32) (Handle, error) { + return SetupDiOpenDevRegKey(deviceInfoSet, DeviceInfoData, Scope, HwProfile, KeyType, samDesired) +} + +//sys setupDiGetDeviceProperty(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, propertyKey *DEVPROPKEY, propertyType *DEVPROPTYPE, propertyBuffer *byte, propertyBufferSize uint32, requiredSize *uint32, flags uint32) (err error) = setupapi.SetupDiGetDevicePropertyW + +// SetupDiGetDeviceProperty function retrieves a specified device instance property. +func SetupDiGetDeviceProperty(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, propertyKey *DEVPROPKEY) (value interface{}, err error) { + reqSize := uint32(256) + for { + var dataType DEVPROPTYPE + buf := make([]byte, reqSize) + err = setupDiGetDeviceProperty(deviceInfoSet, deviceInfoData, propertyKey, &dataType, &buf[0], uint32(len(buf)), &reqSize, 0) + if err == ERROR_INSUFFICIENT_BUFFER { + continue + } + if err != nil { + return + } + switch dataType { + case DEVPROP_TYPE_STRING: + ret := UTF16ToString(bufToUTF16(buf)) + runtime.KeepAlive(buf) + return ret, nil + } + return nil, errors.New("unimplemented property type") + } +} + +//sys setupDiGetDeviceRegistryProperty(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, property SPDRP, propertyRegDataType *uint32, propertyBuffer *byte, propertyBufferSize uint32, requiredSize *uint32) (err error) = setupapi.SetupDiGetDeviceRegistryPropertyW + +// SetupDiGetDeviceRegistryProperty function retrieves a specified Plug and Play device property. +func SetupDiGetDeviceRegistryProperty(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, property SPDRP) (value interface{}, err error) { + reqSize := uint32(256) + for { + var dataType uint32 + buf := make([]byte, reqSize) + err = setupDiGetDeviceRegistryProperty(deviceInfoSet, deviceInfoData, property, &dataType, &buf[0], uint32(len(buf)), &reqSize) + if err == ERROR_INSUFFICIENT_BUFFER { + continue + } + if err != nil { + return + } + return getRegistryValue(buf[:reqSize], dataType) + } +} + +func getRegistryValue(buf []byte, dataType uint32) (interface{}, error) { + switch dataType { + case REG_SZ: + ret := UTF16ToString(bufToUTF16(buf)) + runtime.KeepAlive(buf) + return ret, nil + case REG_EXPAND_SZ: + value := UTF16ToString(bufToUTF16(buf)) + if value == "" { + return "", nil + } + p, err := syscall.UTF16PtrFromString(value) + if err != nil { + return "", err + } + ret := make([]uint16, 100) + for { + n, err := ExpandEnvironmentStrings(p, &ret[0], uint32(len(ret))) + if err != nil { + return "", err + } + if n <= uint32(len(ret)) { + return UTF16ToString(ret[:n]), nil + } + ret = make([]uint16, n) + } + case REG_BINARY: + return buf, nil + case REG_DWORD_LITTLE_ENDIAN: + return binary.LittleEndian.Uint32(buf), nil + case REG_DWORD_BIG_ENDIAN: + return binary.BigEndian.Uint32(buf), nil + case REG_MULTI_SZ: + bufW := bufToUTF16(buf) + a := []string{} + for i := 0; i < len(bufW); { + j := i + wcslen(bufW[i:]) + if i < j { + a = append(a, UTF16ToString(bufW[i:j])) + } + i = j + 1 + } + runtime.KeepAlive(buf) + return a, nil + case REG_QWORD_LITTLE_ENDIAN: + return binary.LittleEndian.Uint64(buf), nil + default: + return nil, fmt.Errorf("Unsupported registry value type: %v", dataType) + } +} + +// bufToUTF16 function reinterprets []byte buffer as []uint16 +func bufToUTF16(buf []byte) []uint16 { + sl := struct { + addr *uint16 + len int + cap int + }{(*uint16)(unsafe.Pointer(&buf[0])), len(buf) / 2, cap(buf) / 2} + return *(*[]uint16)(unsafe.Pointer(&sl)) +} + +// utf16ToBuf function reinterprets []uint16 as []byte +func utf16ToBuf(buf []uint16) []byte { + sl := struct { + addr *byte + len int + cap int + }{(*byte)(unsafe.Pointer(&buf[0])), len(buf) * 2, cap(buf) * 2} + return *(*[]byte)(unsafe.Pointer(&sl)) +} + +func wcslen(str []uint16) int { + for i := 0; i < len(str); i++ { + if str[i] == 0 { + return i + } + } + return len(str) +} + +// DeviceRegistryProperty method retrieves a specified Plug and Play device property. +func (deviceInfoSet DevInfo) DeviceRegistryProperty(deviceInfoData *DevInfoData, property SPDRP) (interface{}, error) { + return SetupDiGetDeviceRegistryProperty(deviceInfoSet, deviceInfoData, property) +} + +//sys setupDiSetDeviceRegistryProperty(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, property SPDRP, propertyBuffer *byte, propertyBufferSize uint32) (err error) = setupapi.SetupDiSetDeviceRegistryPropertyW + +// SetupDiSetDeviceRegistryProperty function sets a Plug and Play device property for a device. +func SetupDiSetDeviceRegistryProperty(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, property SPDRP, propertyBuffers []byte) error { + return setupDiSetDeviceRegistryProperty(deviceInfoSet, deviceInfoData, property, &propertyBuffers[0], uint32(len(propertyBuffers))) +} + +// SetDeviceRegistryProperty function sets a Plug and Play device property for a device. +func (deviceInfoSet DevInfo) SetDeviceRegistryProperty(deviceInfoData *DevInfoData, property SPDRP, propertyBuffers []byte) error { + return SetupDiSetDeviceRegistryProperty(deviceInfoSet, deviceInfoData, property, propertyBuffers) +} + +// SetDeviceRegistryPropertyString method sets a Plug and Play device property string for a device. +func (deviceInfoSet DevInfo) SetDeviceRegistryPropertyString(deviceInfoData *DevInfoData, property SPDRP, str string) error { + str16, err := UTF16FromString(str) + if err != nil { + return err + } + err = SetupDiSetDeviceRegistryProperty(deviceInfoSet, deviceInfoData, property, utf16ToBuf(append(str16, 0))) + runtime.KeepAlive(str16) + return err +} + +//sys setupDiGetDeviceInstallParams(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, deviceInstallParams *DevInstallParams) (err error) = setupapi.SetupDiGetDeviceInstallParamsW + +// SetupDiGetDeviceInstallParams function retrieves device installation parameters for a device information set or a particular device information element. +func SetupDiGetDeviceInstallParams(deviceInfoSet DevInfo, deviceInfoData *DevInfoData) (*DevInstallParams, error) { + params := &DevInstallParams{} + params.size = uint32(unsafe.Sizeof(*params)) + + return params, setupDiGetDeviceInstallParams(deviceInfoSet, deviceInfoData, params) +} + +// DeviceInstallParams method retrieves device installation parameters for a device information set or a particular device information element. +func (deviceInfoSet DevInfo) DeviceInstallParams(deviceInfoData *DevInfoData) (*DevInstallParams, error) { + return SetupDiGetDeviceInstallParams(deviceInfoSet, deviceInfoData) +} + +//sys setupDiGetDeviceInstanceId(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, instanceId *uint16, instanceIdSize uint32, instanceIdRequiredSize *uint32) (err error) = setupapi.SetupDiGetDeviceInstanceIdW + +// SetupDiGetDeviceInstanceId function retrieves the instance ID of the device. +func SetupDiGetDeviceInstanceId(deviceInfoSet DevInfo, deviceInfoData *DevInfoData) (string, error) { + reqSize := uint32(1024) + for { + buf := make([]uint16, reqSize) + err := setupDiGetDeviceInstanceId(deviceInfoSet, deviceInfoData, &buf[0], uint32(len(buf)), &reqSize) + if err == ERROR_INSUFFICIENT_BUFFER { + continue + } + if err != nil { + return "", err + } + return UTF16ToString(buf), nil + } +} + +// DeviceInstanceID method retrieves the instance ID of the device. +func (deviceInfoSet DevInfo) DeviceInstanceID(deviceInfoData *DevInfoData) (string, error) { + return SetupDiGetDeviceInstanceId(deviceInfoSet, deviceInfoData) +} + +// SetupDiGetClassInstallParams function retrieves class installation parameters for a device information set or a particular device information element. +//sys SetupDiGetClassInstallParams(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, classInstallParams *ClassInstallHeader, classInstallParamsSize uint32, requiredSize *uint32) (err error) = setupapi.SetupDiGetClassInstallParamsW + +// ClassInstallParams method retrieves class installation parameters for a device information set or a particular device information element. +func (deviceInfoSet DevInfo) ClassInstallParams(deviceInfoData *DevInfoData, classInstallParams *ClassInstallHeader, classInstallParamsSize uint32, requiredSize *uint32) error { + return SetupDiGetClassInstallParams(deviceInfoSet, deviceInfoData, classInstallParams, classInstallParamsSize, requiredSize) +} + +//sys SetupDiSetDeviceInstallParams(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, deviceInstallParams *DevInstallParams) (err error) = setupapi.SetupDiSetDeviceInstallParamsW + +// SetDeviceInstallParams member sets device installation parameters for a device information set or a particular device information element. +func (deviceInfoSet DevInfo) SetDeviceInstallParams(deviceInfoData *DevInfoData, deviceInstallParams *DevInstallParams) error { + return SetupDiSetDeviceInstallParams(deviceInfoSet, deviceInfoData, deviceInstallParams) +} + +// SetupDiSetClassInstallParams function sets or clears class install parameters for a device information set or a particular device information element. +//sys SetupDiSetClassInstallParams(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, classInstallParams *ClassInstallHeader, classInstallParamsSize uint32) (err error) = setupapi.SetupDiSetClassInstallParamsW + +// SetClassInstallParams method sets or clears class install parameters for a device information set or a particular device information element. +func (deviceInfoSet DevInfo) SetClassInstallParams(deviceInfoData *DevInfoData, classInstallParams *ClassInstallHeader, classInstallParamsSize uint32) error { + return SetupDiSetClassInstallParams(deviceInfoSet, deviceInfoData, classInstallParams, classInstallParamsSize) +} + +//sys setupDiClassNameFromGuidEx(classGUID *GUID, className *uint16, classNameSize uint32, requiredSize *uint32, machineName *uint16, reserved uintptr) (err error) = setupapi.SetupDiClassNameFromGuidExW + +// SetupDiClassNameFromGuidEx function retrieves the class name associated with a class GUID. The class can be installed on a local or remote computer. +func SetupDiClassNameFromGuidEx(classGUID *GUID, machineName string) (className string, err error) { + var classNameUTF16 [MAX_CLASS_NAME_LEN]uint16 + + var machineNameUTF16 *uint16 + if machineName != "" { + machineNameUTF16, err = UTF16PtrFromString(machineName) + if err != nil { + return + } + } + + err = setupDiClassNameFromGuidEx(classGUID, &classNameUTF16[0], MAX_CLASS_NAME_LEN, nil, machineNameUTF16, 0) + if err != nil { + return + } + + className = UTF16ToString(classNameUTF16[:]) + return +} + +//sys setupDiClassGuidsFromNameEx(className *uint16, classGuidList *GUID, classGuidListSize uint32, requiredSize *uint32, machineName *uint16, reserved uintptr) (err error) = setupapi.SetupDiClassGuidsFromNameExW + +// SetupDiClassGuidsFromNameEx function retrieves the GUIDs associated with the specified class name. This resulting list contains the classes currently installed on a local or remote computer. +func SetupDiClassGuidsFromNameEx(className string, machineName string) ([]GUID, error) { + classNameUTF16, err := UTF16PtrFromString(className) + if err != nil { + return nil, err + } + + var machineNameUTF16 *uint16 + if machineName != "" { + machineNameUTF16, err = UTF16PtrFromString(machineName) + if err != nil { + return nil, err + } + } + + reqSize := uint32(4) + for { + buf := make([]GUID, reqSize) + err = setupDiClassGuidsFromNameEx(classNameUTF16, &buf[0], uint32(len(buf)), &reqSize, machineNameUTF16, 0) + if err == ERROR_INSUFFICIENT_BUFFER { + continue + } + if err != nil { + return nil, err + } + return buf[:reqSize], nil + } +} + +//sys setupDiGetSelectedDevice(deviceInfoSet DevInfo, deviceInfoData *DevInfoData) (err error) = setupapi.SetupDiGetSelectedDevice + +// SetupDiGetSelectedDevice function retrieves the selected device information element in a device information set. +func SetupDiGetSelectedDevice(deviceInfoSet DevInfo) (*DevInfoData, error) { + data := &DevInfoData{} + data.size = uint32(unsafe.Sizeof(*data)) + + return data, setupDiGetSelectedDevice(deviceInfoSet, data) +} + +// SelectedDevice method retrieves the selected device information element in a device information set. +func (deviceInfoSet DevInfo) SelectedDevice() (*DevInfoData, error) { + return SetupDiGetSelectedDevice(deviceInfoSet) +} + +// SetupDiSetSelectedDevice function sets a device information element as the selected member of a device information set. This function is typically used by an installation wizard. +//sys SetupDiSetSelectedDevice(deviceInfoSet DevInfo, deviceInfoData *DevInfoData) (err error) = setupapi.SetupDiSetSelectedDevice + +// SetSelectedDevice method sets a device information element as the selected member of a device information set. This function is typically used by an installation wizard. +func (deviceInfoSet DevInfo) SetSelectedDevice(deviceInfoData *DevInfoData) error { + return SetupDiSetSelectedDevice(deviceInfoSet, deviceInfoData) +} + +//sys setupUninstallOEMInf(infFileName *uint16, flags SUOI, reserved uintptr) (err error) = setupapi.SetupUninstallOEMInfW + +// SetupUninstallOEMInf uninstalls the specified driver. +func SetupUninstallOEMInf(infFileName string, flags SUOI) error { + infFileName16, err := UTF16PtrFromString(infFileName) + if err != nil { + return err + } + return setupUninstallOEMInf(infFileName16, flags, 0) +} + +//sys cm_MapCrToWin32Err(configRet CONFIGRET, defaultWin32Error Errno) (ret Errno) = CfgMgr32.CM_MapCrToWin32Err + +//sys cm_Get_Device_Interface_List_Size(len *uint32, interfaceClass *GUID, deviceID *uint16, flags uint32) (ret CONFIGRET) = CfgMgr32.CM_Get_Device_Interface_List_SizeW +//sys cm_Get_Device_Interface_List(interfaceClass *GUID, deviceID *uint16, buffer *uint16, bufferLen uint32, flags uint32) (ret CONFIGRET) = CfgMgr32.CM_Get_Device_Interface_ListW + +func CM_Get_Device_Interface_List(deviceID string, interfaceClass *GUID, flags uint32) ([]string, error) { + deviceID16, err := UTF16PtrFromString(deviceID) + if err != nil { + return nil, err + } + var buf []uint16 + var buflen uint32 + for { + if ret := cm_Get_Device_Interface_List_Size(&buflen, interfaceClass, deviceID16, flags); ret != CR_SUCCESS { + return nil, ret + } + buf = make([]uint16, buflen) + if ret := cm_Get_Device_Interface_List(interfaceClass, deviceID16, &buf[0], buflen, flags); ret == CR_SUCCESS { + break + } else if ret != CR_BUFFER_SMALL { + return nil, ret + } + } + var interfaces []string + for i := 0; i < len(buf); { + j := i + wcslen(buf[i:]) + if i < j { + interfaces = append(interfaces, UTF16ToString(buf[i:j])) + } + i = j + 1 + } + if interfaces == nil { + return nil, ERROR_NO_SUCH_DEVICE_INTERFACE + } + return interfaces, nil +} + +//sys cm_Get_DevNode_Status(status *uint32, problemNumber *uint32, devInst DEVINST, flags uint32) (ret CONFIGRET) = CfgMgr32.CM_Get_DevNode_Status + +func CM_Get_DevNode_Status(status *uint32, problemNumber *uint32, devInst DEVINST, flags uint32) error { + ret := cm_Get_DevNode_Status(status, problemNumber, devInst, flags) + if ret == CR_SUCCESS { + return nil + } + return ret +} diff --git a/vendor/golang.org/x/sys/windows/setupapierrors_windows.go b/vendor/golang.org/x/sys/windows/setupapierrors_windows.go deleted file mode 100644 index 1681810..0000000 --- a/vendor/golang.org/x/sys/windows/setupapierrors_windows.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package windows - -import "syscall" - -const ( - ERROR_EXPECTED_SECTION_NAME syscall.Errno = 0x20000000 | 0xC0000000 | 0 - ERROR_BAD_SECTION_NAME_LINE syscall.Errno = 0x20000000 | 0xC0000000 | 1 - ERROR_SECTION_NAME_TOO_LONG syscall.Errno = 0x20000000 | 0xC0000000 | 2 - ERROR_GENERAL_SYNTAX syscall.Errno = 0x20000000 | 0xC0000000 | 3 - ERROR_WRONG_INF_STYLE syscall.Errno = 0x20000000 | 0xC0000000 | 0x100 - ERROR_SECTION_NOT_FOUND syscall.Errno = 0x20000000 | 0xC0000000 | 0x101 - ERROR_LINE_NOT_FOUND syscall.Errno = 0x20000000 | 0xC0000000 | 0x102 - ERROR_NO_BACKUP syscall.Errno = 0x20000000 | 0xC0000000 | 0x103 - ERROR_NO_ASSOCIATED_CLASS syscall.Errno = 0x20000000 | 0xC0000000 | 0x200 - ERROR_CLASS_MISMATCH syscall.Errno = 0x20000000 | 0xC0000000 | 0x201 - ERROR_DUPLICATE_FOUND syscall.Errno = 0x20000000 | 0xC0000000 | 0x202 - ERROR_NO_DRIVER_SELECTED syscall.Errno = 0x20000000 | 0xC0000000 | 0x203 - ERROR_KEY_DOES_NOT_EXIST syscall.Errno = 0x20000000 | 0xC0000000 | 0x204 - ERROR_INVALID_DEVINST_NAME syscall.Errno = 0x20000000 | 0xC0000000 | 0x205 - ERROR_INVALID_CLASS syscall.Errno = 0x20000000 | 0xC0000000 | 0x206 - ERROR_DEVINST_ALREADY_EXISTS syscall.Errno = 0x20000000 | 0xC0000000 | 0x207 - ERROR_DEVINFO_NOT_REGISTERED syscall.Errno = 0x20000000 | 0xC0000000 | 0x208 - ERROR_INVALID_REG_PROPERTY syscall.Errno = 0x20000000 | 0xC0000000 | 0x209 - ERROR_NO_INF syscall.Errno = 0x20000000 | 0xC0000000 | 0x20A - ERROR_NO_SUCH_DEVINST syscall.Errno = 0x20000000 | 0xC0000000 | 0x20B - ERROR_CANT_LOAD_CLASS_ICON syscall.Errno = 0x20000000 | 0xC0000000 | 0x20C - ERROR_INVALID_CLASS_INSTALLER syscall.Errno = 0x20000000 | 0xC0000000 | 0x20D - ERROR_DI_DO_DEFAULT syscall.Errno = 0x20000000 | 0xC0000000 | 0x20E - ERROR_DI_NOFILECOPY syscall.Errno = 0x20000000 | 0xC0000000 | 0x20F - ERROR_INVALID_HWPROFILE syscall.Errno = 0x20000000 | 0xC0000000 | 0x210 - ERROR_NO_DEVICE_SELECTED syscall.Errno = 0x20000000 | 0xC0000000 | 0x211 - ERROR_DEVINFO_LIST_LOCKED syscall.Errno = 0x20000000 | 0xC0000000 | 0x212 - ERROR_DEVINFO_DATA_LOCKED syscall.Errno = 0x20000000 | 0xC0000000 | 0x213 - ERROR_DI_BAD_PATH syscall.Errno = 0x20000000 | 0xC0000000 | 0x214 - ERROR_NO_CLASSINSTALL_PARAMS syscall.Errno = 0x20000000 | 0xC0000000 | 0x215 - ERROR_FILEQUEUE_LOCKED syscall.Errno = 0x20000000 | 0xC0000000 | 0x216 - ERROR_BAD_SERVICE_INSTALLSECT syscall.Errno = 0x20000000 | 0xC0000000 | 0x217 - ERROR_NO_CLASS_DRIVER_LIST syscall.Errno = 0x20000000 | 0xC0000000 | 0x218 - ERROR_NO_ASSOCIATED_SERVICE syscall.Errno = 0x20000000 | 0xC0000000 | 0x219 - ERROR_NO_DEFAULT_DEVICE_INTERFACE syscall.Errno = 0x20000000 | 0xC0000000 | 0x21A - ERROR_DEVICE_INTERFACE_ACTIVE syscall.Errno = 0x20000000 | 0xC0000000 | 0x21B - ERROR_DEVICE_INTERFACE_REMOVED syscall.Errno = 0x20000000 | 0xC0000000 | 0x21C - ERROR_BAD_INTERFACE_INSTALLSECT syscall.Errno = 0x20000000 | 0xC0000000 | 0x21D - ERROR_NO_SUCH_INTERFACE_CLASS syscall.Errno = 0x20000000 | 0xC0000000 | 0x21E - ERROR_INVALID_REFERENCE_STRING syscall.Errno = 0x20000000 | 0xC0000000 | 0x21F - ERROR_INVALID_MACHINENAME syscall.Errno = 0x20000000 | 0xC0000000 | 0x220 - ERROR_REMOTE_COMM_FAILURE syscall.Errno = 0x20000000 | 0xC0000000 | 0x221 - ERROR_MACHINE_UNAVAILABLE syscall.Errno = 0x20000000 | 0xC0000000 | 0x222 - ERROR_NO_CONFIGMGR_SERVICES syscall.Errno = 0x20000000 | 0xC0000000 | 0x223 - ERROR_INVALID_PROPPAGE_PROVIDER syscall.Errno = 0x20000000 | 0xC0000000 | 0x224 - ERROR_NO_SUCH_DEVICE_INTERFACE syscall.Errno = 0x20000000 | 0xC0000000 | 0x225 - ERROR_DI_POSTPROCESSING_REQUIRED syscall.Errno = 0x20000000 | 0xC0000000 | 0x226 - ERROR_INVALID_COINSTALLER syscall.Errno = 0x20000000 | 0xC0000000 | 0x227 - ERROR_NO_COMPAT_DRIVERS syscall.Errno = 0x20000000 | 0xC0000000 | 0x228 - ERROR_NO_DEVICE_ICON syscall.Errno = 0x20000000 | 0xC0000000 | 0x229 - ERROR_INVALID_INF_LOGCONFIG syscall.Errno = 0x20000000 | 0xC0000000 | 0x22A - ERROR_DI_DONT_INSTALL syscall.Errno = 0x20000000 | 0xC0000000 | 0x22B - ERROR_INVALID_FILTER_DRIVER syscall.Errno = 0x20000000 | 0xC0000000 | 0x22C - ERROR_NON_WINDOWS_NT_DRIVER syscall.Errno = 0x20000000 | 0xC0000000 | 0x22D - ERROR_NON_WINDOWS_DRIVER syscall.Errno = 0x20000000 | 0xC0000000 | 0x22E - ERROR_NO_CATALOG_FOR_OEM_INF syscall.Errno = 0x20000000 | 0xC0000000 | 0x22F - ERROR_DEVINSTALL_QUEUE_NONNATIVE syscall.Errno = 0x20000000 | 0xC0000000 | 0x230 - ERROR_NOT_DISABLEABLE syscall.Errno = 0x20000000 | 0xC0000000 | 0x231 - ERROR_CANT_REMOVE_DEVINST syscall.Errno = 0x20000000 | 0xC0000000 | 0x232 - ERROR_INVALID_TARGET syscall.Errno = 0x20000000 | 0xC0000000 | 0x233 - ERROR_DRIVER_NONNATIVE syscall.Errno = 0x20000000 | 0xC0000000 | 0x234 - ERROR_IN_WOW64 syscall.Errno = 0x20000000 | 0xC0000000 | 0x235 - ERROR_SET_SYSTEM_RESTORE_POINT syscall.Errno = 0x20000000 | 0xC0000000 | 0x236 - ERROR_SCE_DISABLED syscall.Errno = 0x20000000 | 0xC0000000 | 0x238 - ERROR_UNKNOWN_EXCEPTION syscall.Errno = 0x20000000 | 0xC0000000 | 0x239 - ERROR_PNP_REGISTRY_ERROR syscall.Errno = 0x20000000 | 0xC0000000 | 0x23A - ERROR_REMOTE_REQUEST_UNSUPPORTED syscall.Errno = 0x20000000 | 0xC0000000 | 0x23B - ERROR_NOT_AN_INSTALLED_OEM_INF syscall.Errno = 0x20000000 | 0xC0000000 | 0x23C - ERROR_INF_IN_USE_BY_DEVICES syscall.Errno = 0x20000000 | 0xC0000000 | 0x23D - ERROR_DI_FUNCTION_OBSOLETE syscall.Errno = 0x20000000 | 0xC0000000 | 0x23E - ERROR_NO_AUTHENTICODE_CATALOG syscall.Errno = 0x20000000 | 0xC0000000 | 0x23F - ERROR_AUTHENTICODE_DISALLOWED syscall.Errno = 0x20000000 | 0xC0000000 | 0x240 - ERROR_AUTHENTICODE_TRUSTED_PUBLISHER syscall.Errno = 0x20000000 | 0xC0000000 | 0x241 - ERROR_AUTHENTICODE_TRUST_NOT_ESTABLISHED syscall.Errno = 0x20000000 | 0xC0000000 | 0x242 - ERROR_AUTHENTICODE_PUBLISHER_NOT_TRUSTED syscall.Errno = 0x20000000 | 0xC0000000 | 0x243 - ERROR_SIGNATURE_OSATTRIBUTE_MISMATCH syscall.Errno = 0x20000000 | 0xC0000000 | 0x244 - ERROR_ONLY_VALIDATE_VIA_AUTHENTICODE syscall.Errno = 0x20000000 | 0xC0000000 | 0x245 - ERROR_DEVICE_INSTALLER_NOT_READY syscall.Errno = 0x20000000 | 0xC0000000 | 0x246 - ERROR_DRIVER_STORE_ADD_FAILED syscall.Errno = 0x20000000 | 0xC0000000 | 0x247 - ERROR_DEVICE_INSTALL_BLOCKED syscall.Errno = 0x20000000 | 0xC0000000 | 0x248 - ERROR_DRIVER_INSTALL_BLOCKED syscall.Errno = 0x20000000 | 0xC0000000 | 0x249 - ERROR_WRONG_INF_TYPE syscall.Errno = 0x20000000 | 0xC0000000 | 0x24A - ERROR_FILE_HASH_NOT_IN_CATALOG syscall.Errno = 0x20000000 | 0xC0000000 | 0x24B - ERROR_DRIVER_STORE_DELETE_FAILED syscall.Errno = 0x20000000 | 0xC0000000 | 0x24C - ERROR_UNRECOVERABLE_STACK_OVERFLOW syscall.Errno = 0x20000000 | 0xC0000000 | 0x300 - EXCEPTION_SPAPI_UNRECOVERABLE_STACK_OVERFLOW syscall.Errno = ERROR_UNRECOVERABLE_STACK_OVERFLOW - ERROR_NO_DEFAULT_INTERFACE_DEVICE syscall.Errno = ERROR_NO_DEFAULT_DEVICE_INTERFACE - ERROR_INTERFACE_DEVICE_ACTIVE syscall.Errno = ERROR_DEVICE_INTERFACE_ACTIVE - ERROR_INTERFACE_DEVICE_REMOVED syscall.Errno = ERROR_DEVICE_INTERFACE_REMOVED - ERROR_NO_SUCH_INTERFACE_DEVICE syscall.Errno = ERROR_NO_SUCH_DEVICE_INTERFACE -) diff --git a/vendor/golang.org/x/sys/windows/syscall_windows.go b/vendor/golang.org/x/sys/windows/syscall_windows.go index 2ff6aa0..ce3075c 100644 --- a/vendor/golang.org/x/sys/windows/syscall_windows.go +++ b/vendor/golang.org/x/sys/windows/syscall_windows.go @@ -10,6 +10,7 @@ import ( errorspkg "errors" "fmt" "runtime" + "strings" "sync" "syscall" "time" @@ -86,10 +87,8 @@ func StringToUTF16(s string) []uint16 { // s, with a terminating NUL added. If s contains a NUL byte at any // location, it returns (nil, syscall.EINVAL). func UTF16FromString(s string) ([]uint16, error) { - for i := 0; i < len(s); i++ { - if s[i] == 0 { - return nil, syscall.EINVAL - } + if strings.IndexByte(s, 0) != -1 { + return nil, syscall.EINVAL } return utf16.Encode([]rune(s + "\x00")), nil } @@ -186,8 +185,8 @@ func NewCallbackCDecl(fn interface{}) uintptr { //sys GetNamedPipeInfo(pipe Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) //sys GetNamedPipeHandleState(pipe Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW //sys SetNamedPipeHandleState(pipe Handle, state *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32) (err error) = SetNamedPipeHandleState -//sys ReadFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) (err error) -//sys WriteFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) (err error) +//sys readFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) (err error) = ReadFile +//sys writeFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) (err error) = WriteFile //sys GetOverlappedResult(handle Handle, overlapped *Overlapped, done *uint32, wait bool) (err error) //sys SetFilePointer(handle Handle, lowoffset int32, highoffsetptr *int32, whence uint32) (newlowoffset uint32, err error) [failretval==0xffffffff] //sys CloseHandle(handle Handle) (err error) @@ -248,6 +247,7 @@ func NewCallbackCDecl(fn interface{}) uintptr { //sys FreeEnvironmentStrings(envs *uint16) (err error) = kernel32.FreeEnvironmentStringsW //sys GetEnvironmentVariable(name *uint16, buffer *uint16, size uint32) (n uint32, err error) = kernel32.GetEnvironmentVariableW //sys SetEnvironmentVariable(name *uint16, value *uint16) (err error) = kernel32.SetEnvironmentVariableW +//sys ExpandEnvironmentStrings(src *uint16, dst *uint16, size uint32) (n uint32, err error) = kernel32.ExpandEnvironmentStringsW //sys CreateEnvironmentBlock(block **uint16, token Token, inheritExisting bool) (err error) = userenv.CreateEnvironmentBlock //sys DestroyEnvironmentBlock(block *uint16) (err error) = userenv.DestroyEnvironmentBlock //sys getTickCount64() (ms uint64) = kernel32.GetTickCount64 @@ -322,6 +322,8 @@ func NewCallbackCDecl(fn interface{}) uintptr { //sys WriteConsole(console Handle, buf *uint16, towrite uint32, written *uint32, reserved *byte) (err error) = kernel32.WriteConsoleW //sys ReadConsole(console Handle, buf *uint16, toread uint32, read *uint32, inputControl *byte) (err error) = kernel32.ReadConsoleW //sys CreateToolhelp32Snapshot(flags uint32, processId uint32) (handle Handle, err error) [failretval==InvalidHandle] = kernel32.CreateToolhelp32Snapshot +//sys Module32First(snapshot Handle, moduleEntry *ModuleEntry32) (err error) = kernel32.Module32FirstW +//sys Module32Next(snapshot Handle, moduleEntry *ModuleEntry32) (err error) = kernel32.Module32NextW //sys Process32First(snapshot Handle, procEntry *ProcessEntry32) (err error) = kernel32.Process32FirstW //sys Process32Next(snapshot Handle, procEntry *ProcessEntry32) (err error) = kernel32.Process32NextW //sys Thread32First(snapshot Handle, threadEntry *ThreadEntry32) (err error) @@ -360,6 +362,8 @@ func NewCallbackCDecl(fn interface{}) uintptr { //sys SetProcessWorkingSetSizeEx(hProcess Handle, dwMinimumWorkingSetSize uintptr, dwMaximumWorkingSetSize uintptr, flags uint32) (err error) //sys GetCommTimeouts(handle Handle, timeouts *CommTimeouts) (err error) //sys SetCommTimeouts(handle Handle, timeouts *CommTimeouts) (err error) +//sys GetActiveProcessorCount(groupNumber uint16) (ret uint32) +//sys GetMaximumProcessorCount(groupNumber uint16) (ret uint32) // Volume Management Functions //sys DefineDosDevice(flags uint32, deviceName *uint16, targetPath *uint16) (err error) = DefineDosDeviceW @@ -544,12 +548,6 @@ func Read(fd Handle, p []byte) (n int, err error) { } return 0, e } - if raceenabled { - if done > 0 { - raceWriteRange(unsafe.Pointer(&p[0]), int(done)) - } - raceAcquire(unsafe.Pointer(&ioSync)) - } return int(done), nil } @@ -562,12 +560,31 @@ func Write(fd Handle, p []byte) (n int, err error) { if e != nil { return 0, e } - if raceenabled && done > 0 { - raceReadRange(unsafe.Pointer(&p[0]), int(done)) - } return int(done), nil } +func ReadFile(fd Handle, p []byte, done *uint32, overlapped *Overlapped) error { + err := readFile(fd, p, done, overlapped) + if raceenabled { + if *done > 0 { + raceWriteRange(unsafe.Pointer(&p[0]), int(*done)) + } + raceAcquire(unsafe.Pointer(&ioSync)) + } + return err +} + +func WriteFile(fd Handle, p []byte, done *uint32, overlapped *Overlapped) error { + if raceenabled { + raceReleaseMerge(unsafe.Pointer(&ioSync)) + } + err := writeFile(fd, p, done, overlapped) + if raceenabled && *done > 0 { + raceReadRange(unsafe.Pointer(&p[0]), int(*done)) + } + return err +} + var ioSync int64 func Seek(fd Handle, offset int64, whence int) (newoffset int64, err error) { diff --git a/vendor/golang.org/x/sys/windows/types_windows.go b/vendor/golang.org/x/sys/windows/types_windows.go index 286dd1e..e19471c 100644 --- a/vendor/golang.org/x/sys/windows/types_windows.go +++ b/vendor/golang.org/x/sys/windows/types_windows.go @@ -156,6 +156,8 @@ const ( MAX_PATH = 260 MAX_LONG_PATH = 32768 + MAX_MODULE_NAME32 = 255 + MAX_COMPUTERNAME_LENGTH = 15 TIME_ZONE_ID_UNKNOWN = 0 @@ -936,8 +938,8 @@ type StartupInfoEx struct { type ProcThreadAttributeList struct{} type ProcThreadAttributeListContainer struct { - data *ProcThreadAttributeList - heapAllocations []uintptr + data *ProcThreadAttributeList + pointers []unsafe.Pointer } type ProcessInformation struct { @@ -970,6 +972,21 @@ type ThreadEntry32 struct { Flags uint32 } +type ModuleEntry32 struct { + Size uint32 + ModuleID uint32 + ProcessID uint32 + GlblcntUsage uint32 + ProccntUsage uint32 + ModBaseAddr uintptr + ModBaseSize uint32 + ModuleHandle Handle + Module [MAX_MODULE_NAME32 + 1]uint16 + ExePath [MAX_PATH]uint16 +} + +const SizeofModuleEntry32 = unsafe.Sizeof(ModuleEntry32{}) + type Systemtime struct { Year uint16 Month uint16 @@ -2732,6 +2749,43 @@ type PROCESS_BASIC_INFORMATION struct { InheritedFromUniqueProcessId uintptr } +type SYSTEM_PROCESS_INFORMATION struct { + NextEntryOffset uint32 + NumberOfThreads uint32 + WorkingSetPrivateSize int64 + HardFaultCount uint32 + NumberOfThreadsHighWatermark uint32 + CycleTime uint64 + CreateTime int64 + UserTime int64 + KernelTime int64 + ImageName NTUnicodeString + BasePriority int32 + UniqueProcessID uintptr + InheritedFromUniqueProcessID uintptr + HandleCount uint32 + SessionID uint32 + UniqueProcessKey *uint32 + PeakVirtualSize uintptr + VirtualSize uintptr + PageFaultCount uint32 + PeakWorkingSetSize uintptr + WorkingSetSize uintptr + QuotaPeakPagedPoolUsage uintptr + QuotaPagedPoolUsage uintptr + QuotaPeakNonPagedPoolUsage uintptr + QuotaNonPagedPoolUsage uintptr + PagefileUsage uintptr + PeakPagefileUsage uintptr + PrivatePageCount uintptr + ReadOperationCount int64 + WriteOperationCount int64 + OtherOperationCount int64 + ReadTransferCount int64 + WriteTransferCount int64 + OtherTransferCount int64 +} + // SystemInformationClasses for NtQuerySystemInformation and NtSetSystemInformation const ( SystemBasicInformation = iota @@ -3118,3 +3172,5 @@ type ModuleInfo struct { SizeOfImage uint32 EntryPoint uintptr } + +const ALL_PROCESSOR_GROUPS = 0xFFFF diff --git a/vendor/golang.org/x/sys/windows/zsyscall_windows.go b/vendor/golang.org/x/sys/windows/zsyscall_windows.go index 91817d6..68f52c1 100644 --- a/vendor/golang.org/x/sys/windows/zsyscall_windows.go +++ b/vendor/golang.org/x/sys/windows/zsyscall_windows.go @@ -36,6 +36,7 @@ func errnoErr(e syscall.Errno) error { } var ( + modCfgMgr32 = NewLazySystemDLL("CfgMgr32.dll") modadvapi32 = NewLazySystemDLL("advapi32.dll") modcrypt32 = NewLazySystemDLL("crypt32.dll") moddnsapi = NewLazySystemDLL("dnsapi.dll") @@ -48,6 +49,7 @@ var ( modpsapi = NewLazySystemDLL("psapi.dll") modsechost = NewLazySystemDLL("sechost.dll") modsecur32 = NewLazySystemDLL("secur32.dll") + modsetupapi = NewLazySystemDLL("setupapi.dll") modshell32 = NewLazySystemDLL("shell32.dll") moduser32 = NewLazySystemDLL("user32.dll") moduserenv = NewLazySystemDLL("userenv.dll") @@ -56,6 +58,10 @@ var ( modws2_32 = NewLazySystemDLL("ws2_32.dll") modwtsapi32 = NewLazySystemDLL("wtsapi32.dll") + procCM_Get_DevNode_Status = modCfgMgr32.NewProc("CM_Get_DevNode_Status") + procCM_Get_Device_Interface_ListW = modCfgMgr32.NewProc("CM_Get_Device_Interface_ListW") + procCM_Get_Device_Interface_List_SizeW = modCfgMgr32.NewProc("CM_Get_Device_Interface_List_SizeW") + procCM_MapCrToWin32Err = modCfgMgr32.NewProc("CM_MapCrToWin32Err") procAdjustTokenGroups = modadvapi32.NewProc("AdjustTokenGroups") procAdjustTokenPrivileges = modadvapi32.NewProc("AdjustTokenPrivileges") procAllocateAndInitializeSid = modadvapi32.NewProc("AllocateAndInitializeSid") @@ -199,6 +205,7 @@ var ( procDeviceIoControl = modkernel32.NewProc("DeviceIoControl") procDuplicateHandle = modkernel32.NewProc("DuplicateHandle") procExitProcess = modkernel32.NewProc("ExitProcess") + procExpandEnvironmentStringsW = modkernel32.NewProc("ExpandEnvironmentStringsW") procFindClose = modkernel32.NewProc("FindClose") procFindCloseChangeNotification = modkernel32.NewProc("FindCloseChangeNotification") procFindFirstChangeNotificationW = modkernel32.NewProc("FindFirstChangeNotificationW") @@ -219,6 +226,7 @@ var ( procFreeLibrary = modkernel32.NewProc("FreeLibrary") procGenerateConsoleCtrlEvent = modkernel32.NewProc("GenerateConsoleCtrlEvent") procGetACP = modkernel32.NewProc("GetACP") + procGetActiveProcessorCount = modkernel32.NewProc("GetActiveProcessorCount") procGetCommTimeouts = modkernel32.NewProc("GetCommTimeouts") procGetCommandLineW = modkernel32.NewProc("GetCommandLineW") procGetComputerNameExW = modkernel32.NewProc("GetComputerNameExW") @@ -244,6 +252,7 @@ var ( procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") procGetLogicalDrives = modkernel32.NewProc("GetLogicalDrives") procGetLongPathNameW = modkernel32.NewProc("GetLongPathNameW") + procGetMaximumProcessorCount = modkernel32.NewProc("GetMaximumProcessorCount") procGetModuleFileNameW = modkernel32.NewProc("GetModuleFileNameW") procGetModuleHandleExW = modkernel32.NewProc("GetModuleHandleExW") procGetNamedPipeHandleStateW = modkernel32.NewProc("GetNamedPipeHandleStateW") @@ -288,6 +297,8 @@ var ( procLockFileEx = modkernel32.NewProc("LockFileEx") procLockResource = modkernel32.NewProc("LockResource") procMapViewOfFile = modkernel32.NewProc("MapViewOfFile") + procModule32FirstW = modkernel32.NewProc("Module32FirstW") + procModule32NextW = modkernel32.NewProc("Module32NextW") procMoveFileExW = modkernel32.NewProc("MoveFileExW") procMoveFileW = modkernel32.NewProc("MoveFileW") procMultiByteToWideChar = modkernel32.NewProc("MultiByteToWideChar") @@ -400,6 +411,34 @@ var ( procUnsubscribeServiceChangeNotifications = modsechost.NewProc("UnsubscribeServiceChangeNotifications") procGetUserNameExW = modsecur32.NewProc("GetUserNameExW") procTranslateNameW = modsecur32.NewProc("TranslateNameW") + procSetupDiBuildDriverInfoList = modsetupapi.NewProc("SetupDiBuildDriverInfoList") + procSetupDiCallClassInstaller = modsetupapi.NewProc("SetupDiCallClassInstaller") + procSetupDiCancelDriverInfoSearch = modsetupapi.NewProc("SetupDiCancelDriverInfoSearch") + procSetupDiClassGuidsFromNameExW = modsetupapi.NewProc("SetupDiClassGuidsFromNameExW") + procSetupDiClassNameFromGuidExW = modsetupapi.NewProc("SetupDiClassNameFromGuidExW") + procSetupDiCreateDeviceInfoListExW = modsetupapi.NewProc("SetupDiCreateDeviceInfoListExW") + procSetupDiCreateDeviceInfoW = modsetupapi.NewProc("SetupDiCreateDeviceInfoW") + procSetupDiDestroyDeviceInfoList = modsetupapi.NewProc("SetupDiDestroyDeviceInfoList") + procSetupDiDestroyDriverInfoList = modsetupapi.NewProc("SetupDiDestroyDriverInfoList") + procSetupDiEnumDeviceInfo = modsetupapi.NewProc("SetupDiEnumDeviceInfo") + procSetupDiEnumDriverInfoW = modsetupapi.NewProc("SetupDiEnumDriverInfoW") + procSetupDiGetClassDevsExW = modsetupapi.NewProc("SetupDiGetClassDevsExW") + procSetupDiGetClassInstallParamsW = modsetupapi.NewProc("SetupDiGetClassInstallParamsW") + procSetupDiGetDeviceInfoListDetailW = modsetupapi.NewProc("SetupDiGetDeviceInfoListDetailW") + procSetupDiGetDeviceInstallParamsW = modsetupapi.NewProc("SetupDiGetDeviceInstallParamsW") + procSetupDiGetDeviceInstanceIdW = modsetupapi.NewProc("SetupDiGetDeviceInstanceIdW") + procSetupDiGetDevicePropertyW = modsetupapi.NewProc("SetupDiGetDevicePropertyW") + procSetupDiGetDeviceRegistryPropertyW = modsetupapi.NewProc("SetupDiGetDeviceRegistryPropertyW") + procSetupDiGetDriverInfoDetailW = modsetupapi.NewProc("SetupDiGetDriverInfoDetailW") + procSetupDiGetSelectedDevice = modsetupapi.NewProc("SetupDiGetSelectedDevice") + procSetupDiGetSelectedDriverW = modsetupapi.NewProc("SetupDiGetSelectedDriverW") + procSetupDiOpenDevRegKey = modsetupapi.NewProc("SetupDiOpenDevRegKey") + procSetupDiSetClassInstallParamsW = modsetupapi.NewProc("SetupDiSetClassInstallParamsW") + procSetupDiSetDeviceInstallParamsW = modsetupapi.NewProc("SetupDiSetDeviceInstallParamsW") + procSetupDiSetDeviceRegistryPropertyW = modsetupapi.NewProc("SetupDiSetDeviceRegistryPropertyW") + procSetupDiSetSelectedDevice = modsetupapi.NewProc("SetupDiSetSelectedDevice") + procSetupDiSetSelectedDriverW = modsetupapi.NewProc("SetupDiSetSelectedDriverW") + procSetupUninstallOEMInfW = modsetupapi.NewProc("SetupUninstallOEMInfW") procCommandLineToArgvW = modshell32.NewProc("CommandLineToArgvW") procSHGetKnownFolderPath = modshell32.NewProc("SHGetKnownFolderPath") procShellExecuteW = modshell32.NewProc("ShellExecuteW") @@ -447,6 +486,30 @@ var ( procWTSQueryUserToken = modwtsapi32.NewProc("WTSQueryUserToken") ) +func cm_Get_DevNode_Status(status *uint32, problemNumber *uint32, devInst DEVINST, flags uint32) (ret CONFIGRET) { + r0, _, _ := syscall.Syscall6(procCM_Get_DevNode_Status.Addr(), 4, uintptr(unsafe.Pointer(status)), uintptr(unsafe.Pointer(problemNumber)), uintptr(devInst), uintptr(flags), 0, 0) + ret = CONFIGRET(r0) + return +} + +func cm_Get_Device_Interface_List(interfaceClass *GUID, deviceID *uint16, buffer *uint16, bufferLen uint32, flags uint32) (ret CONFIGRET) { + r0, _, _ := syscall.Syscall6(procCM_Get_Device_Interface_ListW.Addr(), 5, uintptr(unsafe.Pointer(interfaceClass)), uintptr(unsafe.Pointer(deviceID)), uintptr(unsafe.Pointer(buffer)), uintptr(bufferLen), uintptr(flags), 0) + ret = CONFIGRET(r0) + return +} + +func cm_Get_Device_Interface_List_Size(len *uint32, interfaceClass *GUID, deviceID *uint16, flags uint32) (ret CONFIGRET) { + r0, _, _ := syscall.Syscall6(procCM_Get_Device_Interface_List_SizeW.Addr(), 4, uintptr(unsafe.Pointer(len)), uintptr(unsafe.Pointer(interfaceClass)), uintptr(unsafe.Pointer(deviceID)), uintptr(flags), 0, 0) + ret = CONFIGRET(r0) + return +} + +func cm_MapCrToWin32Err(configRet CONFIGRET, defaultWin32Error Errno) (ret Errno) { + r0, _, _ := syscall.Syscall(procCM_MapCrToWin32Err.Addr(), 2, uintptr(configRet), uintptr(defaultWin32Error), 0) + ret = Errno(r0) + return +} + func AdjustTokenGroups(token Token, resetToDefault bool, newstate *Tokengroups, buflen uint32, prevstate *Tokengroups, returnlen *uint32) (err error) { var _p0 uint32 if resetToDefault { @@ -1716,6 +1779,15 @@ func ExitProcess(exitcode uint32) { return } +func ExpandEnvironmentStrings(src *uint16, dst *uint16, size uint32) (n uint32, err error) { + r0, _, e1 := syscall.Syscall(procExpandEnvironmentStringsW.Addr(), 3, uintptr(unsafe.Pointer(src)), uintptr(unsafe.Pointer(dst)), uintptr(size)) + n = uint32(r0) + if n == 0 { + err = errnoErr(e1) + } + return +} + func FindClose(handle Handle) (err error) { r1, _, e1 := syscall.Syscall(procFindClose.Addr(), 1, uintptr(handle), 0, 0) if r1 == 0 { @@ -1897,6 +1969,12 @@ func GetACP() (acp uint32) { return } +func GetActiveProcessorCount(groupNumber uint16) (ret uint32) { + r0, _, _ := syscall.Syscall(procGetActiveProcessorCount.Addr(), 1, uintptr(groupNumber), 0, 0) + ret = uint32(r0) + return +} + func GetCommTimeouts(handle Handle, timeouts *CommTimeouts) (err error) { r1, _, e1 := syscall.Syscall(procGetCommTimeouts.Addr(), 2, uintptr(handle), uintptr(unsafe.Pointer(timeouts)), 0) if r1 == 0 { @@ -2099,6 +2177,12 @@ func GetLongPathName(path *uint16, buf *uint16, buflen uint32) (n uint32, err er return } +func GetMaximumProcessorCount(groupNumber uint16) (ret uint32) { + r0, _, _ := syscall.Syscall(procGetMaximumProcessorCount.Addr(), 1, uintptr(groupNumber), 0, 0) + ret = uint32(r0) + return +} + func GetModuleFileName(module Handle, filename *uint16, size uint32) (n uint32, err error) { r0, _, e1 := syscall.Syscall(procGetModuleFileNameW.Addr(), 3, uintptr(module), uintptr(unsafe.Pointer(filename)), uintptr(size)) n = uint32(r0) @@ -2499,6 +2583,22 @@ func MapViewOfFile(handle Handle, access uint32, offsetHigh uint32, offsetLow ui return } +func Module32First(snapshot Handle, moduleEntry *ModuleEntry32) (err error) { + r1, _, e1 := syscall.Syscall(procModule32FirstW.Addr(), 2, uintptr(snapshot), uintptr(unsafe.Pointer(moduleEntry)), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func Module32Next(snapshot Handle, moduleEntry *ModuleEntry32) (err error) { + r1, _, e1 := syscall.Syscall(procModule32NextW.Addr(), 2, uintptr(snapshot), uintptr(unsafe.Pointer(moduleEntry)), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func MoveFileEx(from *uint16, to *uint16, flags uint32) (err error) { r1, _, e1 := syscall.Syscall(procMoveFileExW.Addr(), 3, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags)) if r1 == 0 { @@ -2661,7 +2761,7 @@ func ReadDirectoryChanges(handle Handle, buf *byte, buflen uint32, watchSubTree return } -func ReadFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) (err error) { +func readFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) (err error) { var _p0 *byte if len(buf) > 0 { _p0 = &buf[0] @@ -3103,7 +3203,7 @@ func WriteConsole(console Handle, buf *uint16, towrite uint32, written *uint32, return } -func WriteFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) (err error) { +func writeFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) (err error) { var _p0 *byte if len(buf) > 0 { _p0 = &buf[0] @@ -3432,6 +3532,233 @@ func TranslateName(accName *uint16, accNameFormat uint32, desiredNameFormat uint return } +func SetupDiBuildDriverInfoList(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverType SPDIT) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiBuildDriverInfoList.Addr(), 3, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(driverType)) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func SetupDiCallClassInstaller(installFunction DI_FUNCTION, deviceInfoSet DevInfo, deviceInfoData *DevInfoData) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiCallClassInstaller.Addr(), 3, uintptr(installFunction), uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func SetupDiCancelDriverInfoSearch(deviceInfoSet DevInfo) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiCancelDriverInfoSearch.Addr(), 1, uintptr(deviceInfoSet), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiClassGuidsFromNameEx(className *uint16, classGuidList *GUID, classGuidListSize uint32, requiredSize *uint32, machineName *uint16, reserved uintptr) (err error) { + r1, _, e1 := syscall.Syscall6(procSetupDiClassGuidsFromNameExW.Addr(), 6, uintptr(unsafe.Pointer(className)), uintptr(unsafe.Pointer(classGuidList)), uintptr(classGuidListSize), uintptr(unsafe.Pointer(requiredSize)), uintptr(unsafe.Pointer(machineName)), uintptr(reserved)) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiClassNameFromGuidEx(classGUID *GUID, className *uint16, classNameSize uint32, requiredSize *uint32, machineName *uint16, reserved uintptr) (err error) { + r1, _, e1 := syscall.Syscall6(procSetupDiClassNameFromGuidExW.Addr(), 6, uintptr(unsafe.Pointer(classGUID)), uintptr(unsafe.Pointer(className)), uintptr(classNameSize), uintptr(unsafe.Pointer(requiredSize)), uintptr(unsafe.Pointer(machineName)), uintptr(reserved)) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiCreateDeviceInfoListEx(classGUID *GUID, hwndParent uintptr, machineName *uint16, reserved uintptr) (handle DevInfo, err error) { + r0, _, e1 := syscall.Syscall6(procSetupDiCreateDeviceInfoListExW.Addr(), 4, uintptr(unsafe.Pointer(classGUID)), uintptr(hwndParent), uintptr(unsafe.Pointer(machineName)), uintptr(reserved), 0, 0) + handle = DevInfo(r0) + if handle == DevInfo(InvalidHandle) { + err = errnoErr(e1) + } + return +} + +func setupDiCreateDeviceInfo(deviceInfoSet DevInfo, DeviceName *uint16, classGUID *GUID, DeviceDescription *uint16, hwndParent uintptr, CreationFlags DICD, deviceInfoData *DevInfoData) (err error) { + r1, _, e1 := syscall.Syscall9(procSetupDiCreateDeviceInfoW.Addr(), 7, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(DeviceName)), uintptr(unsafe.Pointer(classGUID)), uintptr(unsafe.Pointer(DeviceDescription)), uintptr(hwndParent), uintptr(CreationFlags), uintptr(unsafe.Pointer(deviceInfoData)), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func SetupDiDestroyDeviceInfoList(deviceInfoSet DevInfo) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiDestroyDeviceInfoList.Addr(), 1, uintptr(deviceInfoSet), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func SetupDiDestroyDriverInfoList(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverType SPDIT) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiDestroyDriverInfoList.Addr(), 3, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(driverType)) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiEnumDeviceInfo(deviceInfoSet DevInfo, memberIndex uint32, deviceInfoData *DevInfoData) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiEnumDeviceInfo.Addr(), 3, uintptr(deviceInfoSet), uintptr(memberIndex), uintptr(unsafe.Pointer(deviceInfoData))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiEnumDriverInfo(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverType SPDIT, memberIndex uint32, driverInfoData *DrvInfoData) (err error) { + r1, _, e1 := syscall.Syscall6(procSetupDiEnumDriverInfoW.Addr(), 5, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(driverType), uintptr(memberIndex), uintptr(unsafe.Pointer(driverInfoData)), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiGetClassDevsEx(classGUID *GUID, Enumerator *uint16, hwndParent uintptr, Flags DIGCF, deviceInfoSet DevInfo, machineName *uint16, reserved uintptr) (handle DevInfo, err error) { + r0, _, e1 := syscall.Syscall9(procSetupDiGetClassDevsExW.Addr(), 7, uintptr(unsafe.Pointer(classGUID)), uintptr(unsafe.Pointer(Enumerator)), uintptr(hwndParent), uintptr(Flags), uintptr(deviceInfoSet), uintptr(unsafe.Pointer(machineName)), uintptr(reserved), 0, 0) + handle = DevInfo(r0) + if handle == DevInfo(InvalidHandle) { + err = errnoErr(e1) + } + return +} + +func SetupDiGetClassInstallParams(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, classInstallParams *ClassInstallHeader, classInstallParamsSize uint32, requiredSize *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procSetupDiGetClassInstallParamsW.Addr(), 5, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(unsafe.Pointer(classInstallParams)), uintptr(classInstallParamsSize), uintptr(unsafe.Pointer(requiredSize)), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiGetDeviceInfoListDetail(deviceInfoSet DevInfo, deviceInfoSetDetailData *DevInfoListDetailData) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiGetDeviceInfoListDetailW.Addr(), 2, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoSetDetailData)), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiGetDeviceInstallParams(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, deviceInstallParams *DevInstallParams) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiGetDeviceInstallParamsW.Addr(), 3, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(unsafe.Pointer(deviceInstallParams))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiGetDeviceInstanceId(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, instanceId *uint16, instanceIdSize uint32, instanceIdRequiredSize *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procSetupDiGetDeviceInstanceIdW.Addr(), 5, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(unsafe.Pointer(instanceId)), uintptr(instanceIdSize), uintptr(unsafe.Pointer(instanceIdRequiredSize)), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiGetDeviceProperty(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, propertyKey *DEVPROPKEY, propertyType *DEVPROPTYPE, propertyBuffer *byte, propertyBufferSize uint32, requiredSize *uint32, flags uint32) (err error) { + r1, _, e1 := syscall.Syscall9(procSetupDiGetDevicePropertyW.Addr(), 8, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(unsafe.Pointer(propertyKey)), uintptr(unsafe.Pointer(propertyType)), uintptr(unsafe.Pointer(propertyBuffer)), uintptr(propertyBufferSize), uintptr(unsafe.Pointer(requiredSize)), uintptr(flags), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiGetDeviceRegistryProperty(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, property SPDRP, propertyRegDataType *uint32, propertyBuffer *byte, propertyBufferSize uint32, requiredSize *uint32) (err error) { + r1, _, e1 := syscall.Syscall9(procSetupDiGetDeviceRegistryPropertyW.Addr(), 7, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(property), uintptr(unsafe.Pointer(propertyRegDataType)), uintptr(unsafe.Pointer(propertyBuffer)), uintptr(propertyBufferSize), uintptr(unsafe.Pointer(requiredSize)), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiGetDriverInfoDetail(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverInfoData *DrvInfoData, driverInfoDetailData *DrvInfoDetailData, driverInfoDetailDataSize uint32, requiredSize *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procSetupDiGetDriverInfoDetailW.Addr(), 6, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(unsafe.Pointer(driverInfoData)), uintptr(unsafe.Pointer(driverInfoDetailData)), uintptr(driverInfoDetailDataSize), uintptr(unsafe.Pointer(requiredSize))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiGetSelectedDevice(deviceInfoSet DevInfo, deviceInfoData *DevInfoData) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiGetSelectedDevice.Addr(), 2, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiGetSelectedDriver(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverInfoData *DrvInfoData) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiGetSelectedDriverW.Addr(), 3, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(unsafe.Pointer(driverInfoData))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func SetupDiOpenDevRegKey(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, Scope DICS_FLAG, HwProfile uint32, KeyType DIREG, samDesired uint32) (key Handle, err error) { + r0, _, e1 := syscall.Syscall6(procSetupDiOpenDevRegKey.Addr(), 6, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(Scope), uintptr(HwProfile), uintptr(KeyType), uintptr(samDesired)) + key = Handle(r0) + if key == InvalidHandle { + err = errnoErr(e1) + } + return +} + +func SetupDiSetClassInstallParams(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, classInstallParams *ClassInstallHeader, classInstallParamsSize uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procSetupDiSetClassInstallParamsW.Addr(), 4, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(unsafe.Pointer(classInstallParams)), uintptr(classInstallParamsSize), 0, 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func SetupDiSetDeviceInstallParams(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, deviceInstallParams *DevInstallParams) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiSetDeviceInstallParamsW.Addr(), 3, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(unsafe.Pointer(deviceInstallParams))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupDiSetDeviceRegistryProperty(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, property SPDRP, propertyBuffer *byte, propertyBufferSize uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procSetupDiSetDeviceRegistryPropertyW.Addr(), 5, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(property), uintptr(unsafe.Pointer(propertyBuffer)), uintptr(propertyBufferSize), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func SetupDiSetSelectedDevice(deviceInfoSet DevInfo, deviceInfoData *DevInfoData) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiSetSelectedDevice.Addr(), 2, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), 0) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func SetupDiSetSelectedDriver(deviceInfoSet DevInfo, deviceInfoData *DevInfoData, driverInfoData *DrvInfoData) (err error) { + r1, _, e1 := syscall.Syscall(procSetupDiSetSelectedDriverW.Addr(), 3, uintptr(deviceInfoSet), uintptr(unsafe.Pointer(deviceInfoData)), uintptr(unsafe.Pointer(driverInfoData))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setupUninstallOEMInf(infFileName *uint16, flags SUOI, reserved uintptr) (err error) { + r1, _, e1 := syscall.Syscall(procSetupUninstallOEMInfW.Addr(), 3, uintptr(unsafe.Pointer(infFileName)), uintptr(flags), uintptr(reserved)) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func CommandLineToArgv(cmd *uint16, argc *int32) (argv *[8192]*[8192]uint16, err error) { r0, _, e1 := syscall.Syscall(procCommandLineToArgvW.Addr(), 2, uintptr(unsafe.Pointer(cmd)), uintptr(unsafe.Pointer(argc)), 0) argv = (*[8192]*[8192]uint16)(unsafe.Pointer(r0)) diff --git a/vendor/modules.txt b/vendor/modules.txt index 9d7e457..409244e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -3,7 +3,7 @@ github.com/google/gopacket github.com/google/gopacket/layers github.com/google/gopacket/pcap -# github.com/gopherjs/gopherjs v0.0.0-20211111143520-d0d5ecc1a356 +# github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96 ## explicit github.com/gopherjs/gopherjs/js # github.com/jtolds/gls v4.20.0+incompatible @@ -19,9 +19,9 @@ github.com/smartystreets/assertions/internal/oglematchers github.com/smartystreets/goconvey/convey github.com/smartystreets/goconvey/convey/gotest github.com/smartystreets/goconvey/convey/reporting -# golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 +# golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 ## explicit -# golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab +# golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f ## explicit golang.org/x/sys/internal/unsafeheader golang.org/x/sys/windows From 372c7ae8c6a3ec1293970f9f9f9e3967296f36f6 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sun, 10 Apr 2022 11:41:19 +0800 Subject: [PATCH 24/34] add chimera build tag --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 09ba13e..e89d6e4 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,12 @@ Hyperscan is a software regular expression matching engine designed with high pe ### Build +`gohs` does not enable the latest api of Hyperscan v5.4 by default, if you want to use it please pass build tags `hyperscan_v54`. + +```bash +go get -u -tags hyperscan_v54 github.com/flier/gohs/hyperscan +``` + `gohs` will use Hyperscan v5 API by default, you can also build for Hyperscan v4 with `hyperscan_v4` tag. ```bash @@ -26,6 +32,7 @@ It is recommended to compile and link Chimera using static libraries. $ mkdir build && cd build $ cmake .. -G Ninja -DBUILD_STATIC_LIBS=on $ ninja && ninja install +$ go get -u -tags chimera github.com/flier/gohs/hyperscan ``` ### Note From b3167f7c947b6dac285daef86afcd0a27846aacd Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sun, 10 Apr 2022 12:17:47 +0800 Subject: [PATCH 25/34] use build tag `hyperscan_v52` for literal support --- .github/workflows/ci.yml | 11 ++ hyperscan/{literal.go => literal_v52.go} | 4 +- .../{literal_test.go => literal_v52_test.go} | 4 +- internal/hs/compile_v5.go | 107 ---------------- internal/hs/compile_v52.go | 116 ++++++++++++++++++ 5 files changed, 131 insertions(+), 111 deletions(-) rename hyperscan/{literal.go => literal_v52.go} (98%) rename hyperscan/{literal_test.go => literal_v52_test.go} (97%) create mode 100644 internal/hs/compile_v52.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c61aa7e..9c6493b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,6 +76,11 @@ jobs: - os: macos-latest hyperscan: 5.2.1 pcre: 8.45 + build_flags: -tags hyperscan_v52,chimera + chimera: true + - os: macos-latest + hyperscan: 5.1.1 + pcre: 8.45 build_flags: -tags chimera chimera: true - os: ubuntu-20.04 @@ -87,6 +92,12 @@ jobs: - os: ubuntu-20.04 hyperscan: 5.2.1 pcre: 8.45 + build_flags: -tags hyperscan_v52,chimera + chimera: true + - os: ubuntu-20.04 + hyperscan: 5.1.1 + pcre: 8.45 + chimera: true - os: ubuntu-18.04 hyperscan: 4.7.0 pcre: 8.41 diff --git a/hyperscan/literal.go b/hyperscan/literal_v52.go similarity index 98% rename from hyperscan/literal.go rename to hyperscan/literal_v52.go index 7e1eb43..e43049c 100644 --- a/hyperscan/literal.go +++ b/hyperscan/literal_v52.go @@ -1,5 +1,5 @@ -//go:build !hyperscan_v4 -// +build !hyperscan_v4 +//go:build hyperscan_v52 || hyperscan_v54 +// +build hyperscan_v52 hyperscan_v54 package hyperscan diff --git a/hyperscan/literal_test.go b/hyperscan/literal_v52_test.go similarity index 97% rename from hyperscan/literal_test.go rename to hyperscan/literal_v52_test.go index de93b1d..7c2152e 100644 --- a/hyperscan/literal_test.go +++ b/hyperscan/literal_v52_test.go @@ -1,5 +1,5 @@ -//go:build !hyperscan_v4 -// +build !hyperscan_v4 +//go:build hyperscan_v52 || hyperscan_v54 +// +build hyperscan_v52 hyperscan_v54 package hyperscan_test diff --git a/internal/hs/compile_v5.go b/internal/hs/compile_v5.go index d7957ac..4ed4fc4 100644 --- a/internal/hs/compile_v5.go +++ b/internal/hs/compile_v5.go @@ -8,11 +8,6 @@ package hs */ import "C" -import ( - "fmt" - "unsafe" -) - const ( // Combination represents logical combination. Combination CompileFlag = C.HS_FLAG_COMBINATION @@ -24,105 +19,3 @@ func init() { CompileFlags['C'] = Combination CompileFlags['Q'] = Quiet } - -// Pure literal is a special case of regular expression. -// A character sequence is regarded as a pure literal if and -// only if each character is read and interpreted independently. -// No syntax association happens between any adjacent characters. -type Literal struct { - Expr string // The expression to parse. - Flags CompileFlag // Flags which modify the behaviour of the expression. - ID int // The ID number to be associated with the corresponding pattern - *ExprInfo -} - -func CompileLit(expression string, flags CompileFlag, mode ModeFlag, info *PlatformInfo) (Database, error) { - var db *C.hs_database_t - var err *C.hs_compile_error_t - var platform *C.hs_platform_info_t - - if info != nil { - platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) - } - - expr := C.CString(expression) - - defer C.free(unsafe.Pointer(expr)) - - ret := C.hs_compile_lit(expr, C.uint(flags), C.ulong(len(expression)), C.uint(mode), platform, &db, &err) - - if err != nil { - defer C.hs_free_compile_error(err) - } - - if ret == C.HS_SUCCESS { - return db, nil - } - - if ret == C.HS_COMPILER_ERROR && err != nil { - return nil, &CompileError{C.GoString(err.message), int(err.expression)} - } - - return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) -} - -type Literals interface { - Literals() []*Literal -} - -func CompileLitMulti(input Literals, mode ModeFlag, info *PlatformInfo) (Database, error) { - var db *C.hs_database_t - var err *C.hs_compile_error_t - var platform *C.hs_platform_info_t - - if info != nil { - platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) - } - - literals := input.Literals() - count := len(literals) - - cexprs := (**C.char)(C.calloc(C.size_t(len(literals)), C.size_t(unsafe.Sizeof(uintptr(0))))) - exprs := (*[1 << 30]*C.char)(unsafe.Pointer(cexprs))[:len(literals):len(literals)] - - clens := (*C.size_t)(C.calloc(C.size_t(len(literals)), C.size_t(unsafe.Sizeof(uintptr(0))))) - lens := (*[1 << 30]C.size_t)(unsafe.Pointer(clens))[:len(literals):len(literals)] - - cflags := (*C.uint)(C.calloc(C.size_t(len(literals)), C.size_t(unsafe.Sizeof(C.uint(0))))) - flags := (*[1 << 30]C.uint)(unsafe.Pointer(cflags))[:len(literals):len(literals)] - - cids := (*C.uint)(C.calloc(C.size_t(len(literals)), C.size_t(unsafe.Sizeof(C.uint(0))))) - ids := (*[1 << 30]C.uint)(unsafe.Pointer(cids))[:len(literals):len(literals)] - - for i, lit := range literals { - exprs[i] = C.CString(lit.Expr) - lens[i] = C.size_t(len(lit.Expr)) - flags[i] = C.uint(lit.Flags) - ids[i] = C.uint(lit.ID) - } - - ret := C.hs_compile_lit_multi(cexprs, cflags, cids, clens, C.uint(count), C.uint(mode), platform, &db, &err) - - for _, expr := range exprs { - C.free(unsafe.Pointer(expr)) - } - - C.free(unsafe.Pointer(cexprs)) - C.free(unsafe.Pointer(clens)) - C.free(unsafe.Pointer(cflags)) - C.free(unsafe.Pointer(cids)) - - if err != nil { - defer C.hs_free_compile_error(err) - } - - if ret == C.HS_SUCCESS { - return db, nil - } - - if ret == C.HS_COMPILER_ERROR && err != nil { - return nil, &CompileError{C.GoString(err.message), int(err.expression)} - } - - return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) -} diff --git a/internal/hs/compile_v52.go b/internal/hs/compile_v52.go new file mode 100644 index 0000000..2e034cd --- /dev/null +++ b/internal/hs/compile_v52.go @@ -0,0 +1,116 @@ +//go:build hyperscan_v52 || hyperscan_v54 +// +build hyperscan_v52 hyperscan_v54 + +package hs + +import ( + "fmt" + "unsafe" +) + +/* +#include +*/ +import "C" + +// Pure literal is a special case of regular expression. +// A character sequence is regarded as a pure literal if and +// only if each character is read and interpreted independently. +// No syntax association happens between any adjacent characters. +type Literal struct { + Expr string // The expression to parse. + Flags CompileFlag // Flags which modify the behaviour of the expression. + ID int // The ID number to be associated with the corresponding pattern + *ExprInfo +} + +func CompileLit(expression string, flags CompileFlag, mode ModeFlag, info *PlatformInfo) (Database, error) { + var db *C.hs_database_t + var err *C.hs_compile_error_t + var platform *C.hs_platform_info_t + + if info != nil { + platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) + } + + expr := C.CString(expression) + + defer C.free(unsafe.Pointer(expr)) + + ret := C.hs_compile_lit(expr, C.uint(flags), C.ulong(len(expression)), C.uint(mode), platform, &db, &err) + + if err != nil { + defer C.hs_free_compile_error(err) + } + + if ret == C.HS_SUCCESS { + return db, nil + } + + if ret == C.HS_COMPILER_ERROR && err != nil { + return nil, &CompileError{C.GoString(err.message), int(err.expression)} + } + + return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) +} + +type Literals interface { + Literals() []*Literal +} + +func CompileLitMulti(input Literals, mode ModeFlag, info *PlatformInfo) (Database, error) { + var db *C.hs_database_t + var err *C.hs_compile_error_t + var platform *C.hs_platform_info_t + + if info != nil { + platform = (*C.struct_hs_platform_info)(unsafe.Pointer(&info.Platform)) + } + + literals := input.Literals() + count := len(literals) + + cexprs := (**C.char)(C.calloc(C.size_t(len(literals)), C.size_t(unsafe.Sizeof(uintptr(0))))) + exprs := (*[1 << 30]*C.char)(unsafe.Pointer(cexprs))[:len(literals):len(literals)] + + clens := (*C.size_t)(C.calloc(C.size_t(len(literals)), C.size_t(unsafe.Sizeof(uintptr(0))))) + lens := (*[1 << 30]C.size_t)(unsafe.Pointer(clens))[:len(literals):len(literals)] + + cflags := (*C.uint)(C.calloc(C.size_t(len(literals)), C.size_t(unsafe.Sizeof(C.uint(0))))) + flags := (*[1 << 30]C.uint)(unsafe.Pointer(cflags))[:len(literals):len(literals)] + + cids := (*C.uint)(C.calloc(C.size_t(len(literals)), C.size_t(unsafe.Sizeof(C.uint(0))))) + ids := (*[1 << 30]C.uint)(unsafe.Pointer(cids))[:len(literals):len(literals)] + + for i, lit := range literals { + exprs[i] = C.CString(lit.Expr) + lens[i] = C.size_t(len(lit.Expr)) + flags[i] = C.uint(lit.Flags) + ids[i] = C.uint(lit.ID) + } + + ret := C.hs_compile_lit_multi(cexprs, cflags, cids, clens, C.uint(count), C.uint(mode), platform, &db, &err) + + for _, expr := range exprs { + C.free(unsafe.Pointer(expr)) + } + + C.free(unsafe.Pointer(cexprs)) + C.free(unsafe.Pointer(clens)) + C.free(unsafe.Pointer(cflags)) + C.free(unsafe.Pointer(cids)) + + if err != nil { + defer C.hs_free_compile_error(err) + } + + if ret == C.HS_SUCCESS { + return db, nil + } + + if ret == C.HS_COMPILER_ERROR && err != nil { + return nil, &CompileError{C.GoString(err.message), int(err.expression)} + } + + return nil, fmt.Errorf("compile error %d, %w", int(ret), ErrCompileError) +} From 6a9dcf419ffbe7e513c6c0b939ae094c4c274f2b Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sun, 10 Apr 2022 12:23:52 +0800 Subject: [PATCH 26/34] fix spell error --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c6493b..e30dc80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04, ubuntu-18.04, macos-latest] - go: [1.18.x 1.17.x, 1.16.x, 1.15.x] + go: [1.18.x, 1.17.x, 1.16.x, 1.15.x] name: Go ${{ matrix.go }} tests @ ${{ matrix.os }} for hyperscan ${{ matrix.hyperscan }} runs-on: ${{ matrix.os }} steps: From 2c64a985eb1326b7609bfebc663f9fd0a5a7d43c Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Sun, 10 Apr 2022 12:25:09 +0800 Subject: [PATCH 27/34] format code --- bench/go/chimera_test.go | 3 ++- bench/go/scan_test.go | 1 + chimera/block.go | 3 ++- chimera/example_runtime_test.go | 3 ++- chimera/runtime_test.go | 3 ++- hyperscan/stream.go | 6 ++++-- internal/ch/compile.go | 3 ++- internal/ch/runtime.go | 6 ++++-- 8 files changed, 19 insertions(+), 9 deletions(-) diff --git a/bench/go/chimera_test.go b/bench/go/chimera_test.go index 2925a34..d784dc5 100644 --- a/bench/go/chimera_test.go +++ b/bench/go/chimera_test.go @@ -26,7 +26,8 @@ func BenchmarkChimeraBlockScan(b *testing.B) { } m := chimera.HandlerFunc(func(id uint, from, to uint64, flags uint, - captured []*chimera.Capture, context interface{}) chimera.Callback { + captured []*chimera.Capture, context interface{}, + ) chimera.Callback { return chimera.Terminate }) diff --git a/bench/go/scan_test.go b/bench/go/scan_test.go index 7df1140..e23289b 100644 --- a/bench/go/scan_test.go +++ b/bench/go/scan_test.go @@ -142,6 +142,7 @@ func BenchmarkHyperscanStreamScan(b *testing.B) { } } } + func BenchmarkRegexpMatch(b *testing.B) { isRaceBuilder := strings.HasSuffix(testenv(), "-race") diff --git a/chimera/block.go b/chimera/block.go index b16967f..e2d8cd4 100644 --- a/chimera/block.go +++ b/chimera/block.go @@ -112,7 +112,8 @@ func newBlockMatcher(scanner *blockScanner) *blockMatcher { } func (m *blockMatcher) OnMatch(id uint, from, to uint64, flags uint, - captured []*Capture, context interface{}) (r Callback) { + captured []*Capture, context interface{}, +) (r Callback) { r = m.MatchRecorder.OnMatch(id, from, to, flags, captured, context) if m.n < 0 { diff --git a/chimera/example_runtime_test.go b/chimera/example_runtime_test.go index 62c4b99..3b40cb0 100644 --- a/chimera/example_runtime_test.go +++ b/chimera/example_runtime_test.go @@ -44,7 +44,8 @@ func ExampleBlockScanner() { var matches []Match handler := chimera.HandlerFunc(func(id uint, from, to uint64, flags uint, - captured []*chimera.Capture, ctx interface{}) chimera.Callback { + captured []*chimera.Capture, ctx interface{}, + ) chimera.Callback { matches = append(matches, Match{from, to}) return chimera.Continue }) diff --git a/chimera/runtime_test.go b/chimera/runtime_test.go index d769872..7499319 100644 --- a/chimera/runtime_test.go +++ b/chimera/runtime_test.go @@ -32,7 +32,8 @@ func TestBlockScanner(t *testing.T) { var matches [][]uint64 matched := func(id uint, from, to uint64, flags uint, - captured []*chimera.Capture, context interface{}) chimera.Callback { + captured []*chimera.Capture, context interface{}, + ) chimera.Callback { matches = append(matches, []uint64{from, to}) return chimera.Continue diff --git a/hyperscan/stream.go b/hyperscan/stream.go index bb674f5..1e02aa6 100644 --- a/hyperscan/stream.go +++ b/hyperscan/stream.go @@ -339,7 +339,8 @@ func (db *streamDatabase) Compress(s Stream) ([]byte, error) { } func (db *streamDatabase) Expand(buf []byte, flags ScanFlag, sc *Scratch, - handler MatchHandler, context interface{}) (Stream, error) { + handler MatchHandler, context interface{}, +) (Stream, error) { var s hs.Stream err := hs.ExpandStream(db.db, &s, buf) @@ -362,7 +363,8 @@ func (db *streamDatabase) Expand(buf []byte, flags ScanFlag, sc *Scratch, } func (db *streamDatabase) ResetAndExpand(s Stream, buf []byte, flags ScanFlag, sc *Scratch, - handler MatchHandler, context interface{}) (Stream, error) { + handler MatchHandler, context interface{}, +) (Stream, error) { ss, ok := s.(*stream) if !ok { return nil, fmt.Errorf("stream %v, %w", s, ErrInvalid) diff --git a/internal/ch/compile.go b/internal/ch/compile.go index c8c4f85..692f3f0 100644 --- a/internal/ch/compile.go +++ b/internal/ch/compile.go @@ -134,7 +134,8 @@ func CompileMulti(p Patterns, mode CompileMode, info *hs.PlatformInfo) (Database // The multiple regular expression compiler. func CompileExtMulti(p Patterns, mode CompileMode, info *hs.PlatformInfo, - matchLimit, matchLimitRecursion uint) (Database, error) { + matchLimit, matchLimitRecursion uint, +) (Database, error) { var db *C.ch_database_t var err *C.ch_compile_error_t var platform *C.hs_platform_info_t diff --git a/internal/ch/runtime.go b/internal/ch/runtime.go index f124676..274c5b5 100644 --- a/internal/ch/runtime.go +++ b/internal/ch/runtime.go @@ -86,7 +86,8 @@ type eventContext struct { //export matchEventCallback func matchEventCallback(id C.uint, from, to C.ulonglong, flags, size C.uint, - capture *C.capture_t, data unsafe.Pointer) C.ch_callback_t { + capture *C.capture_t, data unsafe.Pointer, +) C.ch_callback_t { h := (*handle.Handle)(data) ctx, ok := h.Value().(eventContext) if !ok { @@ -119,7 +120,8 @@ type ScanFlag uint // The block regular expression scanner. func Scan(db Database, data []byte, flags ScanFlag, scratch Scratch, - onEvent MatchEventHandler, onError ErrorEventHandler, context interface{}) error { + onEvent MatchEventHandler, onError ErrorEventHandler, context interface{}, +) error { if data == nil { return ErrInvalid } From f14fec47d8d35130be9beb2228511d4dd88bbf67 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Mon, 11 Apr 2022 09:56:35 +0800 Subject: [PATCH 28/34] upgrade to actions/cache@v3 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e30dc80..514a370 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: go-version: ${{ matrix.go }} - name: Cache Golang modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/.cache/go-build @@ -126,7 +126,7 @@ jobs: - name: Cache Hyperscan library id: cache-hyperscan - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: cache-hyperscan-library with: @@ -164,7 +164,7 @@ jobs: go-version: ${{ matrix.go }} - name: Cache Golang modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/.cache/go-build From 29e26387cbee0f0d0364510bef3330aa24dafd98 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Mon, 11 Apr 2022 10:15:53 +0800 Subject: [PATCH 29/34] use flier/install-hyperscan action to cache hyperscan --- .github/workflows/ci.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 514a370..b396191 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,17 +124,6 @@ jobs: - uses: actions/checkout@v2 - - name: Cache Hyperscan library - id: cache-hyperscan - uses: actions/cache@v3 - env: - cache-name: cache-hyperscan-library - with: - path: ${{ github.workspace }}/dist - key: ${{ runner.os }}-build-hyperscan-${{ matrix.hyperscan }}-pcre-${{ matrix.pcre }} - restore-keys: | - ${{ runner.os }}-build-hyperscan-${{ matrix.hyperscan }}- - - name: Install Hyperscan ${{ matrix.hyperscan }} with PCRE ${{ matrix.pcre }} if: steps.cache-hyperscan.outputs.cache-hit == false uses: flier/install-hyperscan@main @@ -144,6 +133,7 @@ jobs: build_static_lib: on src_dir: ${{ runner.temp }}/hyperscan/ install_prefix: ${{ github.workspace }}/dist/ + cache_key: ${{ runner.os }}-build-hyperscan-${{ matrix.hyperscan }}-pcre-${{ matrix.pcre }} - name: Upload Hyperscan ${{ matrix.hyperscan }} if: steps.cache-hyperscan.outputs.cache-hit == false From 234a807abc019737048675a7c6208e7395a63bac Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Mon, 11 Apr 2022 10:47:03 +0800 Subject: [PATCH 30/34] fix test cases for docker --- hyperscan/platform_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyperscan/platform_test.go b/hyperscan/platform_test.go index a19a06d..0a65132 100644 --- a/hyperscan/platform_test.go +++ b/hyperscan/platform_test.go @@ -13,7 +13,7 @@ func TestPlatform(t *testing.T) { p := hyperscan.PopulatePlatform() So(p, ShouldNotBeNil) - So(p.Tune(), ShouldBeGreaterThan, hyperscan.Generic) + So(p.Tune(), ShouldBeGreaterThanOrEqualTo, hyperscan.Generic) So(p.CpuFeatures(), ShouldBeGreaterThanOrEqualTo, 0) So(p, ShouldResemble, hyperscan.NewPlatform(p.Tune(), p.CpuFeatures())) From 16a18a2cf4592baa2a90a74d67fb11ba99cbd520 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Mon, 11 Apr 2022 10:53:44 +0800 Subject: [PATCH 31/34] upgrade to golangci/golangci-lint-action@v3 --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b396191..d4bbd15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,10 +192,11 @@ jobs: sudo apt-get update sudo apt-get install -yq libhyperscan-dev libpcap-dev + - uses: actions/setup-go@v2 - uses: actions/checkout@v2 - name: golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3 with: version: latest working-directory: hyperscan From 42300bcf0f0f7c743883099f5a5d7afd8ff7e446 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Mon, 11 Apr 2022 11:01:15 +0800 Subject: [PATCH 32/34] fix lint warning --- hyperscan/compile.go | 50 ++++++++++++++++++++++++++------------------ hyperscan/stream.go | 4 ++-- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/hyperscan/compile.go b/hyperscan/compile.go index 31929cb..a08db0c 100644 --- a/hyperscan/compile.go +++ b/hyperscan/compile.go @@ -202,13 +202,15 @@ func (b *DatabaseBuilder) Build() (Database, error) { } // NewBlockDatabase create a block database base on the patterns. -func NewBlockDatabase(patterns ...*Pattern) (BlockDatabase, error) { - db, err := Patterns(patterns).Build(BlockMode) +func NewBlockDatabase(patterns ...*Pattern) (bdb BlockDatabase, err error) { + var db Database + db, err = Patterns(patterns).Build(BlockMode) if err != nil { - return nil, err + return } - return db.(*blockDatabase), err + bdb, _ = db.(*blockDatabase) + return } // NewManagedBlockDatabase is a wrapper for NewBlockDatabase that @@ -228,13 +230,15 @@ func NewManagedBlockDatabase(patterns ...*Pattern) (BlockDatabase, error) { } // NewStreamDatabase create a stream database base on the patterns. -func NewStreamDatabase(patterns ...*Pattern) (StreamDatabase, error) { - db, err := Patterns(patterns).Build(StreamMode) +func NewStreamDatabase(patterns ...*Pattern) (sdb StreamDatabase, err error) { + var db Database + db, err = Patterns(patterns).Build(StreamMode) if err != nil { - return nil, err + return } - return db.(*streamDatabase), err + sdb, _ = db.(*streamDatabase) + return } // NewManagedStreamDatabase is a wrapper for NewStreamDatabase that @@ -254,33 +258,39 @@ func NewManagedStreamDatabase(patterns ...*Pattern) (StreamDatabase, error) { } // NewMediumStreamDatabase create a medium-sized stream database base on the patterns. -func NewMediumStreamDatabase(patterns ...*Pattern) (StreamDatabase, error) { - db, err := Patterns(patterns).Build(StreamMode | SomHorizonMediumMode) +func NewMediumStreamDatabase(patterns ...*Pattern) (sdb StreamDatabase, err error) { + var db Database + db, err = Patterns(patterns).Build(StreamMode | SomHorizonMediumMode) if err != nil { - return nil, err + return } - return db.(*streamDatabase), err + sdb, _ = db.(*streamDatabase) + return } // NewLargeStreamDatabase create a large-sized stream database base on the patterns. -func NewLargeStreamDatabase(patterns ...*Pattern) (StreamDatabase, error) { - db, err := Patterns(patterns).Build(StreamMode | SomHorizonLargeMode) +func NewLargeStreamDatabase(patterns ...*Pattern) (sdb StreamDatabase, err error) { + var db Database + db, err = Patterns(patterns).Build(StreamMode | SomHorizonLargeMode) if err != nil { - return nil, err + return } - return db.(*streamDatabase), err + sdb, _ = db.(*streamDatabase) + return } // NewVectoredDatabase create a vectored database base on the patterns. -func NewVectoredDatabase(patterns ...*Pattern) (VectoredDatabase, error) { - db, err := Patterns(patterns).Build(VectoredMode) +func NewVectoredDatabase(patterns ...*Pattern) (vdb VectoredDatabase, err error) { + var db Database + db, err = Patterns(patterns).Build(VectoredMode) if err != nil { - return nil, err + return } - return db.(*vectoredDatabase), err + vdb, _ = db.(*vectoredDatabase) + return } // Compile a regular expression and returns, if successful, diff --git a/hyperscan/stream.go b/hyperscan/stream.go index 1e02aa6..6dbbb55 100644 --- a/hyperscan/stream.go +++ b/hyperscan/stream.go @@ -175,7 +175,7 @@ func (ss *streamScanner) Scan(reader io.Reader, scratch *Scratch, handler MatchH for { n, err := reader.Read(buf) - if err == io.EOF { + if errors.Is(err, io.EOF) { return nil } @@ -232,7 +232,7 @@ func (m *streamMatcher) scan(reader io.Reader) error { for { n, err := reader.Read(buf) - if err == io.EOF { + if errors.Is(err, io.EOF) { return nil } From f7e2e6cb164d8bb046b3284bcb3a9b0ced9f0190 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Mon, 11 Apr 2022 11:13:34 +0800 Subject: [PATCH 33/34] make golangci-lint happy --- hyperscan/scratch.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hyperscan/scratch.go b/hyperscan/scratch.go index c7c72d1..38cca76 100644 --- a/hyperscan/scratch.go +++ b/hyperscan/scratch.go @@ -43,7 +43,9 @@ func (s *Scratch) Size() (int, error) { return hs.ScratchSize(s.s) } // nolint: // Realloc reallocate the scratch for another database. func (s *Scratch) Realloc(db Database) error { - return hs.ReallocScratch(db.(database).c(), &s.s) // nolint: wrapcheck + r, _ := db.(database) + + return hs.ReallocScratch(r.c(), &s.s) // nolint: wrapcheck } // Clone allocate a scratch space that is a clone of an existing scratch space. From 14128479add76d5cfd594a833f97de5924244060 Mon Sep 17 00:00:00 2001 From: Flier Lu Date: Mon, 11 Apr 2022 11:27:50 +0800 Subject: [PATCH 34/34] add chimera tag for ubuntu 20.04 and hyperscan 5.1.1 --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4bbd15..f834c4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,7 @@ jobs: - os: ubuntu-20.04 hyperscan: 5.1.1 pcre: 8.45 + build_flags: -tags chimera chimera: true - os: ubuntu-18.04 hyperscan: 4.7.0