diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index ece0b0f..0d7c38f 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -13,9 +13,6 @@ jobs: fail-fast: true matrix: go-version: - - 1.13.x - - 1.14.x - - 1.15.x - 1.16.x - 1.17.x os: [ubuntu-latest, macos-latest, windows-latest] diff --git a/README.md b/README.md index 64259e5..2370cec 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ See [CHANGELOG]. - [x] Data driven with `parameterize` mechanism, supporting sequential/random/unique strategies to select data. - [ ] Built-in 100+ commonly used functions for ease, including md5sum, max/min, sleep, gen_random_string etc. - [x] Create and call custom functions with `plugin` mechanism, support [hashicorp plugin] and [go plugin]. -- [ ] Generate html reports with rich test results. +- [x] Generate html reports with rich test results. - [x] Using it as a `CLI tool` or a `library` are both supported. ### Load Testing diff --git a/cli/hrp/cmd/run.go b/cli/hrp/cmd/run.go index 54e32fd..0e4ba33 100644 --- a/cli/hrp/cmd/run.go +++ b/cli/hrp/cmd/run.go @@ -29,6 +29,9 @@ var runCmd = &cobra.Command{ SetDebug(!silentFlag). SetFailfast(!continueOnFailure). SetSaveTests(saveTests) + if genHTMLReport { + runner.GenHTMLReport() + } if proxyUrl != "" { runner.SetProxyUrl(proxyUrl) } @@ -44,6 +47,7 @@ var ( silentFlag bool proxyUrl string saveTests bool + genHTMLReport bool ) func init() { @@ -52,5 +56,5 @@ func init() { runCmd.Flags().BoolVarP(&silentFlag, "silent", "s", false, "disable logging request & response details") runCmd.Flags().StringVarP(&proxyUrl, "proxy-url", "p", "", "set proxy url") runCmd.Flags().BoolVar(&saveTests, "save-tests", false, "save tests summary") - // runCmd.Flags().BoolP("gen-html-report", "r", false, "Generate HTML report") + runCmd.Flags().BoolVarP(&genHTMLReport, "gen-html-report", "r", false, "generate html report") } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8525128..6674459 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,10 +1,11 @@ # Release History -## v0.6.0 (2022-01-27) +## v0.6.0 (2022-02-08) - feat: implement `rendezvous` mechanism for data driven - feat: upload release artifacts to aliyun oss - feat: dump tests summary for execution results +- feat: generate html report for API testing - change: remove sentry sdk ## v0.5.3 (2022-01-25) diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index c6c88e4..688e0c5 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -33,4 +33,4 @@ Copyright 2021 debugtalk * [hrp run](hrp_run.md) - run API test * [hrp startproject](hrp_startproject.md) - create a scaffold project -###### Auto generated by spf13/cobra on 27-Jan-2022 +###### Auto generated by spf13/cobra on 8-Feb-2022 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index daed576..e867df8 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -39,4 +39,4 @@ hrp boom [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 27-Jan-2022 +###### Auto generated by spf13/cobra on 8-Feb-2022 diff --git a/docs/cmd/hrp_har2case.md b/docs/cmd/hrp_har2case.md index 38ef613..fd05af0 100644 --- a/docs/cmd/hrp_har2case.md +++ b/docs/cmd/hrp_har2case.md @@ -23,4 +23,4 @@ hrp har2case $har_path... [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 27-Jan-2022 +###### Auto generated by spf13/cobra on 8-Feb-2022 \ No newline at end of file diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index 0cdb5ca..334cd4f 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -22,6 +22,7 @@ hrp run $path... [flags] ``` --continue-on-failure continue running next step when failure occurs + -r, --gen-html-report generate html report -h, --help help for run -p, --proxy-url string set proxy url --save-tests save tests summary @@ -32,4 +33,4 @@ hrp run $path... [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 27-Jan-2022 +###### Auto generated by spf13/cobra on 8-Feb-2022 diff --git a/docs/cmd/hrp_startproject.md b/docs/cmd/hrp_startproject.md index fbc419b..6bfcd97 100644 --- a/docs/cmd/hrp_startproject.md +++ b/docs/cmd/hrp_startproject.md @@ -16,4 +16,4 @@ hrp startproject $project_name [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 27-Jan-2022 +###### Auto generated by spf13/cobra on 8-Feb-2022 diff --git a/go.mod b/go.mod index 9837f60..1e2d4e5 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/httprunner/hrp -go 1.13 +go 1.16 require ( github.com/denisbrodbeck/machineid v1.0.1 diff --git a/internal/builtin/function.go b/internal/builtin/function.go index ca61adc..e69dd4e 100644 --- a/internal/builtin/function.go +++ b/internal/builtin/function.go @@ -129,3 +129,16 @@ func Dump2YAML(data interface{}, path string) error { } return nil } + +func FormatResponse(raw interface{}) interface{} { + formattedResponse := make(map[string]interface{}) + for key, value := range raw.(map[string]interface{}) { + // convert value to json + if key == "body" { + b, _ := json.MarshalIndent(&value, "", " ") + value = string(b) + } + formattedResponse[key] = value + } + return formattedResponse +} diff --git a/internal/report/template.html b/internal/report/template.html new file mode 100644 index 0000000..978f215 --- /dev/null +++ b/internal/report/template.html @@ -0,0 +1,358 @@ + + + + TestReport + + + + + +

API Test Report

+ +

Summary

+ + + + + + + + + + + + + + + + + + + + + + + + + +
START AT{{.Time.StartAt}}
DURATION{{ .Time.Duration }} seconds
PLATFORMHttpRunnerPlus {{ .Platform.HttprunnerVersion }}{{ .Platform.GoVersion }}{{ .Platform.Platform }}
STATTESTCASES (success/fail)TESTSTEPS (success/fail/error/skip)
total (details) =>{{.Stat.TestCases.Total}} ({{.Stat.TestCases.Success}}/{{.Stat.TestCases.Fail}}){{.Stat.TestSteps.Total}} ({{.Stat.TestSteps.Successes}}/0/{{.Stat.TestSteps.Failures}}/0)
+ +

Details

+{{ range $suite_index, $detail := .Details }} +

{{.Name}}

+ + + + + + + + + + + + + + + {{- range $loop_index, $record := .Records }} + {{- with $record}} + {{- $status := "error"}} + {{- if .Success }} {{ $status = "success" }} {{ end }} + + + + + + + {{- end }} + {{- end }} +
TOTAL: {{.Stat.Total}}SUCCESS: {{.Stat.Successes}}FAILED: 0ERROR: {{.Stat.Failures}}SKIPPED: 0
StatusNameResponse TimeDetail
{{$status}}{{.Name}}{{ .Elapsed }} ms + log + + {{ if .Attachment }} + traceback + + {{- end }} +
+{{- end }} + \ No newline at end of file diff --git a/models.go b/models.go index a6d0a49..7b10ed4 100644 --- a/models.go +++ b/models.go @@ -241,8 +241,8 @@ type platform struct { Platform string `json:"platform" yaml:"platform"` } -// summary stores tests summary for current task execution, maybe include one or multiple testcases -type summary struct { +// Summary stores tests summary for current task execution, maybe include one or multiple testcases +type Summary struct { Success bool `json:"success" yaml:"success"` Stat *stat `json:"stat" yaml:"stat"` Time *testCaseTime `json:"time" yaml:"time"` @@ -250,13 +250,13 @@ type summary struct { Details []*testCaseSummary `json:"details" yaml:"details"` } -func newOutSummary() *summary { +func newOutSummary() *Summary { platForm := &platform{ HttprunnerVersion: version.VERSION, GoVersion: runtime.Version(), Platform: fmt.Sprintf("%v-%v", runtime.GOOS, runtime.GOARCH), } - return &summary{ + return &Summary{ Success: true, Stat: &stat{}, Time: &testCaseTime{ @@ -266,7 +266,7 @@ func newOutSummary() *summary { } } -func (s *summary) appendCaseSummary(caseSummary *testCaseSummary) { +func (s *Summary) appendCaseSummary(caseSummary *testCaseSummary) { s.Success = s.Success && caseSummary.Success s.Stat.TestCases.Total += 1 s.Stat.TestSteps.Total += len(caseSummary.Records) @@ -290,6 +290,7 @@ type stepData struct { Data interface{} `json:"data,omitempty" yaml:"data,omitempty"` // session data or slice of step data ContentSize int64 `json:"content_size" yaml:"content_size"` // response body length ExportVars map[string]interface{} `json:"export_vars,omitempty" yaml:"export_vars,omitempty"` // extract variables + Attachment string `json:"attachment,omitempty" yaml:"attachment,omitempty"` // step error information } type testCaseInOut struct { @@ -302,6 +303,7 @@ type testCaseSummary struct { Name string `json:"name" yaml:"name"` Success bool `json:"success" yaml:"success"` CaseId string `json:"case_id,omitempty" yaml:"case_id,omitempty"` //TODO + Stat *testStepStat `json:"stat" yaml:"stat"` Time *testCaseTime `json:"time" yaml:"time"` InOut *testCaseInOut `json:"in_out" yaml:"in_out"` Log string `json:"log,omitempty" yaml:"log,omitempty"` //TODO @@ -315,7 +317,7 @@ type validationResult struct { } type reqResps struct { - Request *Request `json:"request" yaml:"request"` + Request interface{} `json:"request" yaml:"request"` Response interface{} `json:"response" yaml:"response"` } @@ -334,11 +336,8 @@ type SessionData struct { } func newSessionData() *SessionData { - reqResps := &reqResps{ - Request: &Request{}, - } return &SessionData{ Success: false, - ReqResps: reqResps, + ReqResps: &reqResps{}, } } diff --git a/runner.go b/runner.go index b6f4278..2d976c7 100644 --- a/runner.go +++ b/runner.go @@ -1,10 +1,13 @@ package hrp import ( + "bufio" "bytes" "crypto/tls" + _ "embed" "encoding/json" "fmt" + "html/template" "io/ioutil" "net/http" "net/http/httputil" @@ -30,6 +33,7 @@ import ( const ( summaryPath string = "summary.json" + reportPath string = "report.html" ) // Run starts to run API test with default configs. @@ -44,9 +48,10 @@ func NewRunner(t *testing.T) *HRPRunner { t = &testing.T{} } return &HRPRunner{ - t: t, - failfast: true, // default to failfast - debug: false, // default to turn off debug + t: t, + failfast: true, // default to failfast + debug: false, // default to turn off debug + genHTMLReport: false, client: &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -57,11 +62,12 @@ func NewRunner(t *testing.T) *HRPRunner { } type HRPRunner struct { - t *testing.T - failfast bool - debug bool - saveTests bool - client *http.Client + t *testing.T + failfast bool + debug bool + saveTests bool + genHTMLReport bool + client *http.Client } // SetFailfast configures whether to stop running when one step fails. @@ -100,6 +106,13 @@ func (r *HRPRunner) SetSaveTests(saveTests bool) *HRPRunner { return r } +// GenHTMLReport configures whether to gen html report of api tests. +func (r *HRPRunner) GenHTMLReport() *HRPRunner { + log.Info().Bool("genHTMLReport", true).Msg("[init] SetgenHTMLReport") + r.genHTMLReport = true + return r +} + // Run starts to execute one or multiple testcases. func (r *HRPRunner) Run(testcases ...ITestCase) error { event := ga.EventTracking{ @@ -143,8 +156,16 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error { } s.Time.Duration = time.Since(s.Time.StartAt).Seconds() if r.saveTests { - err = builtin.Dump2JSON(s, summaryPath) - return err + err := builtin.Dump2JSON(s, summaryPath) + if err != nil { + return err + } + } + if r.genHTMLReport { + err := genHTMLReport(s) + if err != nil { + return err + } } } return nil @@ -211,8 +232,17 @@ func (r *caseRunner) run() error { } } if stepData != nil { + if err != nil { + stepData.Attachment = err.Error() + } r.summary.Records = append(r.summary.Records, stepData) r.summary.Success = r.summary.Success && stepData.Success + r.summary.Stat.Total += 1 + if stepData.Success { + r.summary.Stat.Successes += 1 + } else { + r.summary.Stat.Failures += 1 + } } } @@ -558,14 +588,16 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro ContentSize: 0, } sessionData := newSessionData() - if err = copier.Copy(&sessionData.ReqResps.Request, step.Request); err != nil { - log.Error().Err(err).Msg("copy step request data failed") - return - } + + // convert request struct to map + jsonRequest, _ := json.Marshal(&step.Request) + var requestMap map[string]interface{} + _ = json.Unmarshal(jsonRequest, &requestMap) + rawUrl := step.Request.URL method := step.Request.Method req := &http.Request{ - Method: string(method), + Method: method, Header: make(http.Header), Proto: "HTTP/1.1", ProtoMajor: 1, @@ -596,7 +628,7 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro return stepResult, errors.Wrap(err, "parse data failed") } parsedParams := params.(map[string]interface{}) - sessionData.ReqResps.Request.Params = parsedParams + requestMap["params"] = parsedParams if len(parsedParams) > 0 { queryParams = make(url.Values) for k, v := range parsedParams { @@ -628,7 +660,7 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro if err != nil { return stepResult, err } - sessionData.ReqResps.Request.Body = data + requestMap["body"] = data var dataBytes []byte switch vv := data.(type) { case map[string]interface{}: @@ -660,10 +692,11 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro setBodyBytes(req, dataBytes) } // update header - sessionData.ReqResps.Request.Headers = make(map[string]string) + headers := make(map[string]string) for key, value := range req.Header { - sessionData.ReqResps.Request.Headers[key] = value[0] + headers[key] = value[0] } + requestMap["headers"] = headers // prepare url u, err := url.Parse(rawUrl) @@ -709,7 +742,8 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro err = errors.Wrap(err, "init ResponseObject error") return } - sessionData.ReqResps.Response = respObj.respObjMeta + sessionData.ReqResps.Request = requestMap + sessionData.ReqResps.Response = builtin.FormatResponse(respObj.respObjMeta) // extract variables from response extractors := step.Extract @@ -787,6 +821,7 @@ func (r *caseRunner) parseConfig(cfg *TConfig) error { func newSummary() *testCaseSummary { return &testCaseSummary{ Success: true, + Stat: &testStepStat{}, Time: &testCaseTime{}, InOut: &testCaseInOut{}, } @@ -809,3 +844,24 @@ func setBodyBytes(req *http.Request, data []byte) { req.Body = ioutil.NopCloser(bytes.NewReader(data)) req.ContentLength = int64(len(data)) } + +//go:embed internal/report/template.html +var reportTemplate string + +func genHTMLReport(summary *Summary) error { + file, err := os.OpenFile(reportPath, os.O_WRONLY|os.O_CREATE, 0666) + defer file.Close() + if err != nil { + log.Error().Err(err).Msg("open file failed") + return err + } + writer := bufio.NewWriter(file) + tmpl := template.Must(template.New("report").Parse(reportTemplate)) + err = tmpl.Execute(writer, summary) + if err != nil { + log.Error().Err(err).Msg("execute applies a parsed template to the specified data object failed") + return err + } + err = writer.Flush() + return err +} diff --git a/runner_test.go b/runner_test.go index 8996a56..4b79dfc 100644 --- a/runner_test.go +++ b/runner_test.go @@ -120,3 +120,11 @@ func TestInitRendezvous(t *testing.T) { } } } + +func TestGenHTMLReport(t *testing.T) { + summary := newOutSummary() + err := genHTMLReport(summary) + if err != nil { + t.Error(err) + } +}