Merge pull request #59 from httprunner/go-plugin

refactor go plugin

- feat: add github.com/httprunner/hrp/plugin sub-module
- feat: add scaffold for plugin
- feat: catch Interrupt and SIGTERM signals to ensure plugin quitted
- change: report event for initializing plugin
- doc: add plugin docs
- refactor: plugin structure

README.md

@@ -16,7 +16,7 @@
 - [x] Testcases can be described in multiple formats, `YAML`/`JSON`/`Golang`, and they are interchangeable.
 - [x] With [`HAR`][HAR] support, you can use Charles/Fiddler/Chrome/etc as a script recording generator.
 - [x] Supports `variables`/`extract`/`validate`/`hooks` mechanisms to create extremely complex test scenarios.
-- [ ] Built-in integration of rich functions, and you can also use [`go plugin`][plugin] to create and call custom functions.
+- [x] Built-in integration of rich functions, and you can also use [hashicorp/plugin] or [go plugin] to create and call custom functions.
 - [x] Inherit all powerful features of [`Boomer`][Boomer] and [`locust`][locust], you can run `load test` without extra work.
 - [x] Using it as a `CLI tool` or a `library` are both supported.
 
@@ -272,7 +272,8 @@ func TestCaseDemo(t *testing.T) {
 [jmespath]: https://jmespath.org/
 [allure]: https://docs.qameta.io/allure/
 [HAR]: http://httparchive.org/
-[plugin]: https://pkg.go.dev/plugin
+[hashicorp/plugin]: https://github.com/hashicorp/go-plugin
+[go plugin]: https://pkg.go.dev/plugin
 [demo.json]: https://github.com/httprunner/hrp/blob/main/examples/demo.json
 [examples]: https://github.com/httprunner/hrp/blob/main/examples/
 [CHANGELOG]: docs/CHANGELOG.md

boomer.go

@@ -1,26 +1,31 @@
 package hrp
 
 import (
+	"sync"
 	"time"
 
 	"github.com/jinzhu/copier"
 	"github.com/rs/zerolog/log"
 
 	"github.com/httprunner/hrp/internal/boomer"
 	"github.com/httprunner/hrp/internal/ga"
+	"github.com/httprunner/hrp/plugin/common"
 )
 
 func NewBoomer(spawnCount int, spawnRate float64) *HRPBoomer {
 	b := &HRPBoomer{
-		Boomer: boomer.NewStandaloneBoomer(spawnCount, spawnRate),
-		debug:  false,
+		Boomer:       boomer.NewStandaloneBoomer(spawnCount, spawnRate),
+		pluginsMutex: new(sync.RWMutex),
+		debug:        false,
 	}
 	return b
 }
 
 type HRPBoomer struct {
 	*boomer.Boomer
-	debug bool
+	plugins      []common.Plugin // each task has its own plugin process
+	pluginsMutex *sync.RWMutex   // avoid data race
+	debug        bool
 }
 
 // SetDebug configures whether to log HTTP request and response content.
@@ -57,14 +62,34 @@ func (b *HRPBoomer) Run(testcases ...ITestCase) {
 	b.Boomer.Run(taskSlice...)
 }
 
+func (b *HRPBoomer) Quit() {
+	b.pluginsMutex.Lock()
+	plugins := b.plugins
+	b.pluginsMutex.Unlock()
+	for _, plugin := range plugins {
+		plugin.Quit()
+	}
+	b.Boomer.Quit()
+}
+
 func (b *HRPBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task {
 	hrpRunner := NewRunner(nil).SetDebug(b.debug)
 	config := testcase.Config.ToStruct()
+
+	// each testcase has its own plugin process
+	plugin, _ := initPlugin(config.Path)
+	if plugin != nil {
+		b.pluginsMutex.Lock()
+		b.plugins = append(b.plugins, plugin)
+		b.pluginsMutex.Unlock()
+	}
+
 	return &boomer.Task{
 		Name:   config.Name,
 		Weight: config.Weight,
 		Fn: func() {
 			runner := hrpRunner.newCaseRunner(testcase)
+			runner.parser.plugin = plugin
 
 			testcaseSuccess := true       // flag whole testcase result
 			var transactionSuccess = true // flag current transaction result
@@ -74,13 +99,21 @@ func (b *HRPBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task {
 			// copy config to avoid data racing
 			if err := copier.Copy(caseConfig, cfg); err != nil {
 				log.Error().Err(err).Msg("copy config data failed")
+				return
 			}
 			// iterate through all parameter iterators and update case variables
 			for _, it := range caseConfig.ParametersSetting.Iterators {
 				if it.HasNext() {
 					caseConfig.Variables = mergeVariables(it.Next(), caseConfig.Variables)
 				}
 			}
+
+			config := runner.TestCase.Config
+			if err := runner.parseConfig(config); err != nil {
+				log.Error().Err(err).Msg("parse config failed")
+				return
+			}
+
 			startTime := time.Now()
 			for index, step := range testcase.TestSteps {
 				stepData, err := runner.runStep(index, caseConfig)

boomer_test.go

@@ -6,6 +6,9 @@ import (
 )
 
 func TestBoomerStandaloneRun(t *testing.T) {
+	buildHashicorpPlugin()
+	defer removeHashicorpPlugin()
+
 	testcase1 := &TestCase{
 		Config: NewConfig("TestCase1").SetBaseURL("http://httpbin.org"),
 		TestSteps: []IStep{

docs/CHANGELOG.md

@@ -1,8 +1,11 @@
 # Release History
 
-## v0.5.2 (2022-01-16)
+## v0.5.2 (2022-01-19)
 
 - feat: support creating and calling custom functions with [hashicorp/go-plugin](https://github.com/hashicorp/go-plugin)
+- feat: add scaffold demo with hashicorp plugin
+- feat: report events for initializing plugin
+- fix: log failures when the assertion failed
 
 ## v0.5.1 (2022-01-13)
 

docs/cmd/hrp.md

@@ -33,4 +33,4 @@ Copyright 2021 debugtalk
 * [hrp run](hrp_run.md)	 - run API test
 * [hrp startproject](hrp_startproject.md)	 - create a scaffold project
 
-###### Auto generated by spf13/cobra on 17-Jan-2022
+###### Auto generated by spf13/cobra on 18-Jan-2022

docs/cmd/hrp_boom.md

@@ -39,4 +39,4 @@ hrp boom [flags]
 
 * [hrp](hrp.md)	 - One-stop solution for HTTP(S) testing.
 
-###### Auto generated by spf13/cobra on 17-Jan-2022
+###### Auto generated by spf13/cobra on 18-Jan-2022

docs/cmd/hrp_har2case.md

@@ -23,4 +23,4 @@ hrp har2case $har_path... [flags]
 
 * [hrp](hrp.md)	 - One-stop solution for HTTP(S) testing.
 
-###### Auto generated by spf13/cobra on 17-Jan-2022
+###### Auto generated by spf13/cobra on 18-Jan-2022

docs/cmd/hrp_run.md

@@ -31,4 +31,4 @@ hrp run $path... [flags]
 
 * [hrp](hrp.md)	 - One-stop solution for HTTP(S) testing.
 
-###### Auto generated by spf13/cobra on 17-Jan-2022
+###### Auto generated by spf13/cobra on 18-Jan-2022

docs/cmd/hrp_startproject.md

@@ -16,4 +16,4 @@ hrp startproject $project_name [flags]
 
 * [hrp](hrp.md)	 - One-stop solution for HTTP(S) testing.
 
-###### Auto generated by spf13/cobra on 17-Jan-2022
+###### Auto generated by spf13/cobra on 18-Jan-2022

examples/demo.json

@@ -3,9 +3,9 @@
         "name": "demo with complex mechanisms",
         "base_url": "https://postman-echo.com",
         "variables": {
-            "a": 12.3,
+            "a": "${sum(10, 2.3)}",
             "b": 3.45,
-            "n": 5,
+            "n": "${sum_ints(1, 2, 2)}",
             "varFoo1": "${gen_random_string($n)}",
             "varFoo2": "${max($a, $b)}"
         }

examples/demo.yaml

@@ -2,9 +2,9 @@ config:
     name: demo with complex mechanisms
     base_url: https://postman-echo.com
     variables:
-        a: 12.3
+        a: ${sum(10, 2.3)}
         b: 3.45
-        "n": 5
+        "n": ${sum_ints(1, 2, 2)}
         varFoo1: ${gen_random_string($n)}
         varFoo2: ${max($a, $b)}
 teststeps:

go.mod

@@ -6,8 +6,7 @@ require (
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/getsentry/sentry-go v0.11.0
 	github.com/google/uuid v1.3.0
-	github.com/hashicorp/go-hclog v1.1.0
-	github.com/hashicorp/go-plugin v1.4.3
+	github.com/httprunner/hrp/plugin v0.0.0-00010101000000-000000000000
 	github.com/jinzhu/copier v0.3.2
 	github.com/jmespath/go-jmespath v0.4.0
 	github.com/maja42/goval v1.2.1
@@ -21,3 +20,5 @@ require (
 	golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
+
+replace github.com/httprunner/hrp/plugin => ./plugin

internal/boomer/stats.go

@@ -66,7 +66,7 @@ func (s *requestStats) logTransaction(name string, success bool, responseTime in
 		s.transactionPassed++
 	} else {
 		s.transactionFailed++
-		s.get(name, "transaction").logError("")
+		s.get(name, "transaction").logFailures()
 	}
 	s.get(name, "transaction").log(responseTime, contentLength)
 }
@@ -77,8 +77,8 @@ func (s *requestStats) logRequest(method, name string, responseTime int64, conte
 }
 
 func (s *requestStats) logError(method, name, err string) {
-	s.total.logError(err)
-	s.get(name, method).logError(err)
+	s.total.logFailures()
+	s.get(name, method).logFailures()
 
 	// store error in errors map
 	key := genMD5(method, name, err)
@@ -264,7 +264,7 @@ func (s *statsEntry) logResponseTime(responseTime int64) {
 	}
 }
 
-func (s *statsEntry) logError(err string) {
+func (s *statsEntry) logFailures() {
 	s.NumFailures++
 	key := time.Now().Unix()
 	_, ok := s.NumFailPerSec[key]

internal/scaffold/demo.go

@@ -6,8 +6,8 @@ var demoTestCase = &hrp.TestCase{
 	Config: hrp.NewConfig("demo with complex mechanisms").
 		SetBaseURL("https://postman-echo.com").
 		WithVariables(map[string]interface{}{ // global level variables
-			"n":       5,
-			"a":       12.3,
+			"n":       "${sum_ints(1, 2, 2)}",
+			"a":       "${sum(10, 2.3)}",
 			"b":       3.45,
 			"varFoo1": "${gen_random_string($n)}",
 			"varFoo2": "${max($a, $b)}", // 12.3; eval with built-in function
@@ -56,6 +56,50 @@ var demoTestCase = &hrp.TestCase{
 	},
 }
 
+// debugtalk.go
+var demoPlugin = `package main
+
+import (
+	"fmt"
+
+	"github.com/httprunner/hrp/plugin"
+)
+
+func SumTwoInt(a, b int) int {
+	return a + b
+}
+
+func SumInts(args ...int) int {
+	var sum int
+	for _, arg := range args {
+		sum += arg
+	}
+	return sum
+}
+
+func Sum(args ...interface{}) (interface{}, error) {
+	var sum float64
+	for _, arg := range args {
+		switch v := arg.(type) {
+		case int:
+			sum += float64(v)
+		case float64:
+			sum += v
+		default:
+			return nil, fmt.Errorf("unexpected type: %T", arg)
+		}
+	}
+	return sum, nil
+}
+
+func main() {
+	plugin.Register("sum_ints", SumInts)
+	plugin.Register("sum_two_int", SumTwoInt)
+	plugin.Register("sum", Sum)
+	plugin.Serve()
+}
+`
+
 // .gitignore
 var demoIgnoreContent = `.env
 reports/*
@@ -64,9 +108,13 @@ reports/*
 .idea/
 .DS_Store
 output/
+
+# plugin
+debugtalk.bin
+debugtalk.so
 `
 
 // .env
 var demoEnvContent = `USERNAME=debugtalk
-"PASSWORD=123456
+PASSWORD=123456
 `

internal/scaffold/demo_test.go

@@ -2,8 +2,12 @@ package scaffold
 
 import (
 	"fmt"
+	"os"
+	"os/exec"
 	"testing"
 
+	"github.com/rs/zerolog/log"
+
 	"github.com/httprunner/hrp"
 )
 
@@ -12,6 +16,21 @@ var (
 	demoTestCaseYAMLPath = "../../examples/demo.yaml"
 )
 
+func buildHashicorpPlugin() {
+	log.Info().Msg("[init] build hashicorp go plugin")
+	cmd := exec.Command("go", "build",
+		"-o", "../../examples/debugtalk.bin",
+		"../../examples/plugin/hashicorp.go", "../../examples/plugin/debugtalk.go")
+	if err := cmd.Run(); err != nil {
+		panic(err)
+	}
+}
+
+func removeHashicorpPlugin() {
+	log.Info().Msg("[teardown] remove hashicorp plugin")
+	os.Remove("../../examples/debugtalk.bin")
+}
+
 func TestGenDemoTestCase(t *testing.T) {
 	tCase, _ := demoTestCase.ToTCase()
 	err := tCase.Dump2JSON(demoTestCaseJSONPath)
@@ -25,13 +44,20 @@ func TestGenDemoTestCase(t *testing.T) {
 }
 
 func Example_demo() {
+	buildHashicorpPlugin()
+	defer removeHashicorpPlugin()
+
+	demoTestCase.Config.ToStruct().Path = "../../examples/debugtalk.bin"
 	err := hrp.NewRunner(nil).Run(demoTestCase) // hrp.Run(demoTestCase)
 	fmt.Println(err)
 	// Output:
 	// <nil>
 }
 
 func Example_jsonDemo() {
+	buildHashicorpPlugin()
+	defer removeHashicorpPlugin()
+
 	testCase := &hrp.TestCasePath{Path: demoTestCaseJSONPath}
 	err := hrp.NewRunner(nil).Run(testCase) // hrp.Run(testCase)
 	fmt.Println(err)
@@ -40,6 +66,9 @@ func Example_jsonDemo() {
 }
 
 func Example_yamlDemo() {
+	buildHashicorpPlugin()
+	defer removeHashicorpPlugin()
+
 	testCase := &hrp.TestCasePath{Path: demoTestCaseYAMLPath}
 	err := hrp.NewRunner(nil).Run(testCase) // hrp.Run(testCase)
 	fmt.Println(err)