Merge pull request #75 from xucong053/main

feat: generate html reports for API testing #74
This commit is contained in:
debugtalk
2022-02-08 17:02:34 +08:00
committed by GitHub
15 changed files with 479 additions and 42 deletions

View File

@@ -13,9 +13,6 @@ jobs:
fail-fast: true
matrix:
go-version:
- 1.13.x
- 1.14.x
- 1.15.x
- 1.16.x
- 1.17.x
os: [ubuntu-latest, macos-latest, windows-latest]

View File

@@ -23,7 +23,7 @@ See [CHANGELOG].
- [x] Data driven with `parameterize` mechanism, supporting sequential/random/unique strategies to select data.
- [ ] Built-in 100+ commonly used functions for ease, including md5sum, max/min, sleep, gen_random_string etc.
- [x] Create and call custom functions with `plugin` mechanism, support [hashicorp plugin] and [go plugin].
- [ ] Generate html reports with rich test results.
- [x] Generate html reports with rich test results.
- [x] Using it as a `CLI tool` or a `library` are both supported.
### Load Testing

View File

@@ -29,6 +29,9 @@ var runCmd = &cobra.Command{
SetDebug(!silentFlag).
SetFailfast(!continueOnFailure).
SetSaveTests(saveTests)
if genHTMLReport {
runner.GenHTMLReport()
}
if proxyUrl != "" {
runner.SetProxyUrl(proxyUrl)
}
@@ -44,6 +47,7 @@ var (
silentFlag bool
proxyUrl string
saveTests bool
genHTMLReport bool
)
func init() {
@@ -52,5 +56,5 @@ func init() {
runCmd.Flags().BoolVarP(&silentFlag, "silent", "s", false, "disable logging request & response details")
runCmd.Flags().StringVarP(&proxyUrl, "proxy-url", "p", "", "set proxy url")
runCmd.Flags().BoolVar(&saveTests, "save-tests", false, "save tests summary")
// runCmd.Flags().BoolP("gen-html-report", "r", false, "Generate HTML report")
runCmd.Flags().BoolVarP(&genHTMLReport, "gen-html-report", "r", false, "generate html report")
}

View File

@@ -1,10 +1,11 @@
# Release History
## v0.6.0 (2022-01-27)
## v0.6.0 (2022-02-08)
- feat: implement `rendezvous` mechanism for data driven
- feat: upload release artifacts to aliyun oss
- feat: dump tests summary for execution results
- feat: generate html report for API testing
- change: remove sentry sdk
## v0.5.3 (2022-01-25)

View File

@@ -33,4 +33,4 @@ Copyright 2021 debugtalk
* [hrp run](hrp_run.md) - run API test
* [hrp startproject](hrp_startproject.md) - create a scaffold project
###### Auto generated by spf13/cobra on 27-Jan-2022
###### Auto generated by spf13/cobra on 8-Feb-2022

View File

@@ -39,4 +39,4 @@ hrp boom [flags]
* [hrp](hrp.md) - One-stop solution for HTTP(S) testing.
###### Auto generated by spf13/cobra on 27-Jan-2022
###### Auto generated by spf13/cobra on 8-Feb-2022

View File

@@ -23,4 +23,4 @@ hrp har2case $har_path... [flags]
* [hrp](hrp.md) - One-stop solution for HTTP(S) testing.
###### Auto generated by spf13/cobra on 27-Jan-2022
###### Auto generated by spf13/cobra on 8-Feb-2022

View File

@@ -22,6 +22,7 @@ hrp run $path... [flags]
```
--continue-on-failure continue running next step when failure occurs
-r, --gen-html-report generate html report
-h, --help help for run
-p, --proxy-url string set proxy url
--save-tests save tests summary
@@ -32,4 +33,4 @@ hrp run $path... [flags]
* [hrp](hrp.md) - One-stop solution for HTTP(S) testing.
###### Auto generated by spf13/cobra on 27-Jan-2022
###### Auto generated by spf13/cobra on 8-Feb-2022

View File

@@ -16,4 +16,4 @@ hrp startproject $project_name [flags]
* [hrp](hrp.md) - One-stop solution for HTTP(S) testing.
###### Auto generated by spf13/cobra on 27-Jan-2022
###### Auto generated by spf13/cobra on 8-Feb-2022

2
go.mod
View File

@@ -1,6 +1,6 @@
module github.com/httprunner/hrp
go 1.13
go 1.16
require (
github.com/denisbrodbeck/machineid v1.0.1

View File

@@ -129,3 +129,16 @@ func Dump2YAML(data interface{}, path string) error {
}
return nil
}
func FormatResponse(raw interface{}) interface{} {
formattedResponse := make(map[string]interface{})
for key, value := range raw.(map[string]interface{}) {
// convert value to json
if key == "body" {
b, _ := json.MarshalIndent(&value, "", " ")
value = string(b)
}
formattedResponse[key] = value
}
return formattedResponse
}

View File

@@ -0,0 +1,358 @@
<head>
<meta content="text/html; charset=utf-8" http-equiv="content-type"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>TestReport</title>
<style>
body {
background-color: #f2f2f2;
color: #333;
margin: 0 auto;
width: 960px;
}
#summary {
width: 960px;
margin-bottom: 20px;
}
#summary th {
background-color: skyblue;
padding: 5px 12px;
}
#summary td {
background-color: lightblue;
text-align: center;
padding: 4px 8px;
}
.details {
width: 960px;
margin-bottom: 20px;
}
.details th {
background-color: skyblue;
padding: 5px 12px;
}
.details tr .passed {
background-color: lightgreen;
}
.details tr .failed {
background-color: red;
}
.details tr .unchecked {
background-color: gray;
}
.details td {
background-color: lightblue;
padding: 5px 12px;
}
.details .detail {
background-color: lightgrey;
font-size: smaller;
padding: 5px 10px;
line-height: 20px;
text-align: left;
}
.details .success {
background-color: greenyellow;
}
.details .error {
background-color: red;
}
.details .failure {
background-color: salmon;
}
.details .skipped {
background-color: gray;
}
.button {
font-size: 1em;
padding: 6px;
width: 4em;
text-align: center;
background-color: #06d85f;
border-radius: 20px/50px;
cursor: pointer;
transition: all 0.3s ease-out;
}
a.button {
color: gray;
text-decoration: none;
display: inline-block;
}
.button:hover {
background: #2cffbd;
}
.overlay {
position: fixed;
top: 0;
bottom: 0;
left: 0;
right: 0;
background: rgba(0, 0, 0, 0.7);
transition: opacity 500ms;
visibility: hidden;
opacity: 0;
line-height: 25px;
}
.overlay:target {
visibility: visible;
opacity: 1;
}
.popup {
margin: 70px auto;
padding: 20px;
background: #fff;
border-radius: 10px;
width: 50%;
position: relative;
transition: all 3s ease-in-out;
}
.popup h2 {
margin-top: 0;
color: #333;
font-family: Tahoma, Arial, sans-serif;
}
.popup .close {
position: absolute;
top: 20px;
right: 30px;
transition: all 200ms;
font-size: 30px;
font-weight: bold;
text-decoration: none;
color: #333;
}
.popup .close:hover {
color: #06d85f;
}
.popup .content {
max-height: 80%;
overflow: auto;
text-align: left;
}
.popup .separator {
color: royalblue
}
@media screen and (max-width: 700px) {
.box {
width: 70%;
}
.popup {
width: 70%;
}
}
</style>
</head>
<body>
<h1>API Test Report</h1>
<h2>Summary</h2>
<table id="summary">
<tr>
<th>START AT</th>
<td colspan="4">{{.Time.StartAt}}</td>
</tr>
<tr>
<th>DURATION</th>
<td colspan="4">{{ .Time.Duration }} seconds</td>
</tr>
<tr>
<th>PLATFORM</th>
<td>HttpRunnerPlus {{ .Platform.HttprunnerVersion }}</td>
<td>{{ .Platform.GoVersion }}</td>
<td colspan="2">{{ .Platform.Platform }}</td>
</tr>
<tr>
<th>STAT</th>
<th colspan="2">TESTCASES (success/fail)</th>
<th colspan="2">TESTSTEPS (success/fail/error/skip)</th>
</tr>
<tr>
<td>total (details) =></td>
<td colspan="2">{{.Stat.TestCases.Total}} ({{.Stat.TestCases.Success}}/{{.Stat.TestCases.Fail}})</td>
<td colspan="2">{{.Stat.TestSteps.Total}} ({{.Stat.TestSteps.Successes}}/0/{{.Stat.TestSteps.Failures}}/0)</td>
</tr>
</table>
<h2>Details</h2>
{{ range $suite_index, $detail := .Details }}
<h3>{{.Name}}</h3>
<table id="suite_{{$suite_index}}" class="details">
<tr>
<td>TOTAL: {{.Stat.Total}}</td>
<td>SUCCESS: {{.Stat.Successes}}</td>
<td>FAILED: 0</td>
<td>ERROR: {{.Stat.Failures}}</td>
<td>SKIPPED: 0</td>
</tr>
<tr>
<th>Status</th>
<th colspan="2">Name</th>
<th>Response Time</th>
<th>Detail</th>
</tr>
{{- range $loop_index, $record := .Records }}
{{- with $record}}
{{- $status := "error"}}
{{- if .Success }} {{ $status = "success" }} {{ end }}
<tr id="record_{{$suite_index}}_{{$loop_index}}">
<th class={{$status}} style="width:5em;">{{$status}}</th>
<td colspan="2">{{.Name}}</td>
<td style="text-align:center;width:6em;">{{ .Elapsed }} ms</td>
<td class="detail">
<a class="button" href="#popup_log_{{$suite_index}}_{{$loop_index}}">log</a>
<div id="popup_log_{{$suite_index}}_{{$loop_index}}" class="overlay">
<div class="popup">
<h2>Request and Response data</h2>
<a class="close" href="#record_{{$suite_index}}_{{$loop_index}}">&times;</a>
<div class="content">
<h3>Name: {{ .Name }}</h3>
<h3>Request:</h3>
<div style="overflow: auto">
<table>
{{- range $key, $value := .Data.ReqResps.Request}}
<tr>
<th>{{$key}}</th>
<td align="left">
{{- if eq $key "headers" }}
{{- range $k, $v := $value }}
<pre>{{$k}}: {{$v}}</pre>
{{- end -}}
{{- else if eq $key "params" }}
{{- range $k, $v := $value }}
<pre>{{$k}}: {{$v}}</pre>
{{- end -}}
{{- else if eq $key "cookies" }}
{{- range $k, $v := $value }}
<pre>{{$k}}: {{$v}}</pre>
{{- end -}}
{{- else }}
<pre>{{$value}}</pre>
{{- end }}
</td>
</tr>
{{- end }}
</table>
</div>
<h3>Response:</h3>
<div style="overflow: auto">
<table>
{{- range $key, $value := .Data.ReqResps.Response}}
<tr>
<th>{{$key}}</th>
<td align="left">
{{- if eq $key "headers" }}
{{- range $k, $v := $value}}
<pre>{{$k}}: {{$v}}</pre>
{{- end -}}
{{- else if eq $key "cookies" }}
{{- range $k, $v := $value }}
<pre>{{$k}}: {{$v}}</pre>
{{- end -}}
{{- else }}
<pre>{{ $value }}</pre>
{{- end }}
</td>
</tr>
{{- end }}
</table>
</div>
<h3>Validators:</h3>
<div style="overflow: auto">
{{- if .Data.Validators }}
<table>
<tr>
<th>check</th>
<th>comparator</th>
<th>expect value</th>
<th>actual value</th>
</tr>
{{- range $validator := .Data.Validators }}
<tr>
{{- if eq $validator.CheckResult "pass" }}
<td class="passed">
{{- else if eq $validator.CheckResult "fail" }}
<td class="failed">
{{- else if eq $validator.CheckResult "unchecked" }}
<td class="unchecked">
{{- end }}
{{$validator.Check}}
</td>
<td>{{$validator.Assert}}</td>
<td>{{$validator.Expect}}</td>
<td>{{$validator.CheckValue}}</td>
</tr>
{{- end }}
</table>
{{- end }}
<h3>Statistics:</h3>
<div style="overflow: auto">
<table>
<tr>
<th>content_size(bytes)</th>
<td>{{ .ContentSize }}</td>
</tr>
<tr>
<th>response_time(ms)</th>
<td>{{ .Elapsed }}</td>
</tr>
<tr>
<th>elapsed(ms)</th>
<td>{{ .Elapsed }}</td>
</tr>
</table>
</div>
</div>
</div>
</div>
</div>
{{ if .Attachment }}
<a class="button" href="#popup_attachment_{{$suite_index}}_{{$loop_index}}">traceback</a>
<div id="popup_attachment_{{$suite_index}}_{{$loop_index}}" class="overlay">
<div class="popup">
<h2>Traceback Message</h2>
<a class="close" href="#record_{{$suite_index}}_{{$loop_index}}">&times;</a>
<div class="content">
<pre>{{ .Attachment }}</pre>
</div>
</div>
</div>
{{- end }}
</td>
</tr>
{{- end }}
{{- end }}
</table>
{{- end }}
</body>

View File

@@ -241,8 +241,8 @@ type platform struct {
Platform string `json:"platform" yaml:"platform"`
}
// summary stores tests summary for current task execution, maybe include one or multiple testcases
type summary struct {
// Summary stores tests summary for current task execution, maybe include one or multiple testcases
type Summary struct {
Success bool `json:"success" yaml:"success"`
Stat *stat `json:"stat" yaml:"stat"`
Time *testCaseTime `json:"time" yaml:"time"`
@@ -250,13 +250,13 @@ type summary struct {
Details []*testCaseSummary `json:"details" yaml:"details"`
}
func newOutSummary() *summary {
func newOutSummary() *Summary {
platForm := &platform{
HttprunnerVersion: version.VERSION,
GoVersion: runtime.Version(),
Platform: fmt.Sprintf("%v-%v", runtime.GOOS, runtime.GOARCH),
}
return &summary{
return &Summary{
Success: true,
Stat: &stat{},
Time: &testCaseTime{
@@ -266,7 +266,7 @@ func newOutSummary() *summary {
}
}
func (s *summary) appendCaseSummary(caseSummary *testCaseSummary) {
func (s *Summary) appendCaseSummary(caseSummary *testCaseSummary) {
s.Success = s.Success && caseSummary.Success
s.Stat.TestCases.Total += 1
s.Stat.TestSteps.Total += len(caseSummary.Records)
@@ -290,6 +290,7 @@ type stepData struct {
Data interface{} `json:"data,omitempty" yaml:"data,omitempty"` // session data or slice of step data
ContentSize int64 `json:"content_size" yaml:"content_size"` // response body length
ExportVars map[string]interface{} `json:"export_vars,omitempty" yaml:"export_vars,omitempty"` // extract variables
Attachment string `json:"attachment,omitempty" yaml:"attachment,omitempty"` // step error information
}
type testCaseInOut struct {
@@ -302,6 +303,7 @@ type testCaseSummary struct {
Name string `json:"name" yaml:"name"`
Success bool `json:"success" yaml:"success"`
CaseId string `json:"case_id,omitempty" yaml:"case_id,omitempty"` //TODO
Stat *testStepStat `json:"stat" yaml:"stat"`
Time *testCaseTime `json:"time" yaml:"time"`
InOut *testCaseInOut `json:"in_out" yaml:"in_out"`
Log string `json:"log,omitempty" yaml:"log,omitempty"` //TODO
@@ -315,7 +317,7 @@ type validationResult struct {
}
type reqResps struct {
Request *Request `json:"request" yaml:"request"`
Request interface{} `json:"request" yaml:"request"`
Response interface{} `json:"response" yaml:"response"`
}
@@ -334,11 +336,8 @@ type SessionData struct {
}
func newSessionData() *SessionData {
reqResps := &reqResps{
Request: &Request{},
}
return &SessionData{
Success: false,
ReqResps: reqResps,
ReqResps: &reqResps{},
}
}

View File

@@ -1,10 +1,13 @@
package hrp
import (
"bufio"
"bytes"
"crypto/tls"
_ "embed"
"encoding/json"
"fmt"
"html/template"
"io/ioutil"
"net/http"
"net/http/httputil"
@@ -30,6 +33,7 @@ import (
const (
summaryPath string = "summary.json"
reportPath string = "report.html"
)
// Run starts to run API test with default configs.
@@ -44,9 +48,10 @@ func NewRunner(t *testing.T) *HRPRunner {
t = &testing.T{}
}
return &HRPRunner{
t: t,
failfast: true, // default to failfast
debug: false, // default to turn off debug
t: t,
failfast: true, // default to failfast
debug: false, // default to turn off debug
genHTMLReport: false,
client: &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
@@ -57,11 +62,12 @@ func NewRunner(t *testing.T) *HRPRunner {
}
type HRPRunner struct {
t *testing.T
failfast bool
debug bool
saveTests bool
client *http.Client
t *testing.T
failfast bool
debug bool
saveTests bool
genHTMLReport bool
client *http.Client
}
// SetFailfast configures whether to stop running when one step fails.
@@ -100,6 +106,13 @@ func (r *HRPRunner) SetSaveTests(saveTests bool) *HRPRunner {
return r
}
// GenHTMLReport configures whether to gen html report of api tests.
func (r *HRPRunner) GenHTMLReport() *HRPRunner {
log.Info().Bool("genHTMLReport", true).Msg("[init] SetgenHTMLReport")
r.genHTMLReport = true
return r
}
// Run starts to execute one or multiple testcases.
func (r *HRPRunner) Run(testcases ...ITestCase) error {
event := ga.EventTracking{
@@ -143,8 +156,16 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error {
}
s.Time.Duration = time.Since(s.Time.StartAt).Seconds()
if r.saveTests {
err = builtin.Dump2JSON(s, summaryPath)
return err
err := builtin.Dump2JSON(s, summaryPath)
if err != nil {
return err
}
}
if r.genHTMLReport {
err := genHTMLReport(s)
if err != nil {
return err
}
}
}
return nil
@@ -211,8 +232,17 @@ func (r *caseRunner) run() error {
}
}
if stepData != nil {
if err != nil {
stepData.Attachment = err.Error()
}
r.summary.Records = append(r.summary.Records, stepData)
r.summary.Success = r.summary.Success && stepData.Success
r.summary.Stat.Total += 1
if stepData.Success {
r.summary.Stat.Successes += 1
} else {
r.summary.Stat.Failures += 1
}
}
}
@@ -558,14 +588,16 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro
ContentSize: 0,
}
sessionData := newSessionData()
if err = copier.Copy(&sessionData.ReqResps.Request, step.Request); err != nil {
log.Error().Err(err).Msg("copy step request data failed")
return
}
// convert request struct to map
jsonRequest, _ := json.Marshal(&step.Request)
var requestMap map[string]interface{}
_ = json.Unmarshal(jsonRequest, &requestMap)
rawUrl := step.Request.URL
method := step.Request.Method
req := &http.Request{
Method: string(method),
Method: method,
Header: make(http.Header),
Proto: "HTTP/1.1",
ProtoMajor: 1,
@@ -596,7 +628,7 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro
return stepResult, errors.Wrap(err, "parse data failed")
}
parsedParams := params.(map[string]interface{})
sessionData.ReqResps.Request.Params = parsedParams
requestMap["params"] = parsedParams
if len(parsedParams) > 0 {
queryParams = make(url.Values)
for k, v := range parsedParams {
@@ -628,7 +660,7 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro
if err != nil {
return stepResult, err
}
sessionData.ReqResps.Request.Body = data
requestMap["body"] = data
var dataBytes []byte
switch vv := data.(type) {
case map[string]interface{}:
@@ -660,10 +692,11 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro
setBodyBytes(req, dataBytes)
}
// update header
sessionData.ReqResps.Request.Headers = make(map[string]string)
headers := make(map[string]string)
for key, value := range req.Header {
sessionData.ReqResps.Request.Headers[key] = value[0]
headers[key] = value[0]
}
requestMap["headers"] = headers
// prepare url
u, err := url.Parse(rawUrl)
@@ -709,7 +742,8 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro
err = errors.Wrap(err, "init ResponseObject error")
return
}
sessionData.ReqResps.Response = respObj.respObjMeta
sessionData.ReqResps.Request = requestMap
sessionData.ReqResps.Response = builtin.FormatResponse(respObj.respObjMeta)
// extract variables from response
extractors := step.Extract
@@ -787,6 +821,7 @@ func (r *caseRunner) parseConfig(cfg *TConfig) error {
func newSummary() *testCaseSummary {
return &testCaseSummary{
Success: true,
Stat: &testStepStat{},
Time: &testCaseTime{},
InOut: &testCaseInOut{},
}
@@ -809,3 +844,24 @@ func setBodyBytes(req *http.Request, data []byte) {
req.Body = ioutil.NopCloser(bytes.NewReader(data))
req.ContentLength = int64(len(data))
}
//go:embed internal/report/template.html
var reportTemplate string
func genHTMLReport(summary *Summary) error {
file, err := os.OpenFile(reportPath, os.O_WRONLY|os.O_CREATE, 0666)
defer file.Close()
if err != nil {
log.Error().Err(err).Msg("open file failed")
return err
}
writer := bufio.NewWriter(file)
tmpl := template.Must(template.New("report").Parse(reportTemplate))
err = tmpl.Execute(writer, summary)
if err != nil {
log.Error().Err(err).Msg("execute applies a parsed template to the specified data object failed")
return err
}
err = writer.Flush()
return err
}

View File

@@ -120,3 +120,11 @@ func TestInitRendezvous(t *testing.T) {
}
}
}
func TestGenHTMLReport(t *testing.T) {
summary := newOutSummary()
err := genHTMLReport(summary)
if err != nil {
t.Error(err)
}
}