From e87b1023baef488751e450f66ed3f3ec4e7cf119 Mon Sep 17 00:00:00 2001 From: xucong053 Date: Mon, 14 Feb 2022 16:41:18 +0800 Subject: [PATCH] fix: modify logging logic of summary. --- docs/cmd/hrp.md | 2 +- docs/cmd/hrp_boom.md | 2 +- docs/cmd/hrp_har2case.md | 2 +- docs/cmd/hrp_run.md | 2 +- docs/cmd/hrp_startproject.md | 2 +- examples/extract_test.go | 1 - examples/request_test.go | 1 - examples/validate_test.go | 1 - examples/variables_test.go | 3 -- internal/builtin/function.go | 70 ++++++++++++++++++++++++++++++++ internal/scaffold/main.go | 57 +++++--------------------- runner.go | 79 ++++++++++++++++++++++++------------ runner_test.go | 15 +++++-- step_test.go | 1 - 14 files changed, 151 insertions(+), 87 deletions(-) diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index 688e0c52..2a01d308 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -33,4 +33,4 @@ Copyright 2021 debugtalk * [hrp run](hrp_run.md) - run API test * [hrp startproject](hrp_startproject.md) - create a scaffold project -###### Auto generated by spf13/cobra on 8-Feb-2022 +###### Auto generated by spf13/cobra on 14-Feb-2022 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index e867df88..914f747f 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -39,4 +39,4 @@ hrp boom [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 8-Feb-2022 +###### Auto generated by spf13/cobra on 14-Feb-2022 diff --git a/docs/cmd/hrp_har2case.md b/docs/cmd/hrp_har2case.md index fd05af09..504d6524 100644 --- a/docs/cmd/hrp_har2case.md +++ b/docs/cmd/hrp_har2case.md @@ -23,4 +23,4 @@ hrp har2case $har_path... [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 8-Feb-2022 \ No newline at end of file +###### Auto generated by spf13/cobra on 14-Feb-2022 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index 334cd4fd..7c6ca58c 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -33,4 +33,4 @@ hrp run $path... [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 8-Feb-2022 +###### Auto generated by spf13/cobra on 14-Feb-2022 diff --git a/docs/cmd/hrp_startproject.md b/docs/cmd/hrp_startproject.md index 6bfcd978..1894cd0b 100644 --- a/docs/cmd/hrp_startproject.md +++ b/docs/cmd/hrp_startproject.md @@ -16,4 +16,4 @@ hrp startproject $project_name [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 8-Feb-2022 +###### Auto generated by spf13/cobra on 14-Feb-2022 diff --git a/examples/extract_test.go b/examples/extract_test.go index ec72277d..0d1184bd 100644 --- a/examples/extract_test.go +++ b/examples/extract_test.go @@ -62,7 +62,6 @@ func TestCaseExtractStepAssociation(t *testing.T) { WithJmesPath("body.args.foo1", "varFoo1"). Validate(). AssertEqual("$statusCode", 200, "check status code"). - AssertEqual("headers.Connection", "keep-alive", "check header Connection"). AssertEqual("$contentType", "application/json; charset=utf-8", "check header Content-Type"). AssertEqual("$varFoo1", "bar1", "check args foo1"). AssertEqual("body.args.foo2", "bar2", "check args foo2"). diff --git a/examples/request_test.go b/examples/request_test.go index 6312df07..2a8a2383 100644 --- a/examples/request_test.go +++ b/examples/request_test.go @@ -20,7 +20,6 @@ func TestCaseBasicRequest(t *testing.T) { }). Validate(). AssertEqual("status_code", 200, "check status code"). - AssertEqual("headers.Connection", "keep-alive", "check header Connection"). AssertEqual("headers.\"Content-Type\"", "application/json; charset=utf-8", "check header Content-Type"). AssertEqual("body.args.foo1", "bar1", "check args foo1"). AssertEqual("body.args.foo2", "bar2", "check args foo2"), diff --git a/examples/validate_test.go b/examples/validate_test.go index 24d60e25..95ff040b 100644 --- a/examples/validate_test.go +++ b/examples/validate_test.go @@ -25,7 +25,6 @@ func TestCaseValidateStep(t *testing.T) { WithJmesPath("body.args.foo1", "varFoo1"). Validate(). AssertEqual("status_code", "$expectedStatusCode", "check status code"). // assert status code - AssertEqual("headers.Connection", "keep-alive", "check header Connection"). // assert response header AssertEqual("headers.\"Content-Type\"", "application/json; charset=utf-8", "check header Content-Type"). // assert response header, with double quotes AssertEqual("body.args.foo1", "bar1", "check args foo1"). // assert response json body with jmespath AssertEqual("body.args.foo2", "bar2", "check args foo2"). diff --git a/examples/variables_test.go b/examples/variables_test.go index 9fb4c0fb..d1f856fc 100644 --- a/examples/variables_test.go +++ b/examples/variables_test.go @@ -22,7 +22,6 @@ func TestCaseConfigVariables(t *testing.T) { WithHeaders(map[string]string{"User-Agent": "$agent"}). Validate(). AssertEqual("status_code", "$expectedStatusCode", "check status code"). - AssertEqual("headers.Connection", "keep-alive", "check header Connection"). AssertEqual("headers.\"Content-Type\"", "application/json; charset=utf-8", "check header Content-Type"). AssertEqual("body.args.foo1", "bar1", "check args foo1"). AssertEqual("body.args.foo2", "bar2", "check args foo2"). @@ -53,7 +52,6 @@ func TestCaseStepVariables(t *testing.T) { WithHeaders(map[string]string{"User-Agent": "$agent"}). Validate(). AssertEqual("status_code", "$expectedStatusCode", "check status code"). - AssertEqual("headers.Connection", "keep-alive", "check header Connection"). AssertEqual("headers.\"Content-Type\"", "application/json; charset=utf-8", "check header Content-Type"). AssertEqual("body.args.foo1", "bar1", "check args foo1"). AssertEqual("body.args.foo2", "bar2", "check args foo2"). @@ -88,7 +86,6 @@ func TestCaseOverrideConfigVariables(t *testing.T) { WithHeaders(map[string]string{"User-Agent": "$agent"}). Validate(). AssertEqual("status_code", "$expectedStatusCode", "check status code"). - AssertEqual("headers.Connection", "keep-alive", "check header Connection"). AssertEqual("headers.\"Content-Type\"", "application/json; charset=utf-8", "check header Content-Type"). AssertEqual("body.args.foo1", "bar1", "check args foo1"). AssertEqual("body.args.foo2", "bar2", "check args foo2"). diff --git a/internal/builtin/function.go b/internal/builtin/function.go index e69dd4ec..434b082d 100644 --- a/internal/builtin/function.go +++ b/internal/builtin/function.go @@ -6,9 +6,12 @@ import ( "encoding/csv" "encoding/hex" "encoding/json" + "fmt" "io/ioutil" "math" "math/rand" + "os" + "os/exec" "path/filepath" "strings" "time" @@ -142,3 +145,70 @@ func FormatResponse(raw interface{}) interface{} { } return formattedResponse } + +func ExecCommand(cmd *exec.Cmd, cwd string) error { + log.Info().Str("cmd", cmd.String()).Str("cwd", cwd).Msg("exec command") + cmd.Dir = cwd + output, err := cmd.CombinedOutput() + out := strings.TrimSpace(string(output)) + if err != nil { + log.Error().Err(err).Str("output", out).Msg("exec command failed") + } else if len(out) != 0 { + log.Info().Str("output", out).Msg("exec command success") + } + return err +} + +func CreateFolder(folderPath string) error { + log.Info().Str("path", folderPath).Msg("create folder") + err := os.MkdirAll(folderPath, os.ModePerm) + if err != nil { + log.Error().Err(err).Msg("create folder failed") + return err + } + return nil +} + +func CreateFile(filePath string, data string) error { + log.Info().Str("path", filePath).Msg("create file") + err := ioutil.WriteFile(filePath, []byte(data), 0o644) + if err != nil { + log.Error().Err(err).Msg("create file failed") + return err + } + return nil +} + +// isFilePathExists returns true if path exists, whether path is file or dir +func isPathExists(path string) bool { + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + return true +} + +// isFilePathExists returns true if path exists and path is file +func isFilePathExists(path string) bool { + info, err := os.Stat(path) + if err != nil { + // path not exists + return false + } + + // path exists + if info.IsDir() { + // path is dir, not file + return false + } + return true +} + +func EnsureFolderExists(folderPath string) error { + if !isPathExists(folderPath) { + err := CreateFolder(folderPath) + return err + } else if isFilePathExists(folderPath) { + return fmt.Errorf("path %v should be directory", folderPath) + } + return nil +} diff --git a/internal/scaffold/main.go b/internal/scaffold/main.go index 04fd84d8..6f2511a3 100644 --- a/internal/scaffold/main.go +++ b/internal/scaffold/main.go @@ -2,11 +2,9 @@ package scaffold import ( "fmt" - "io/ioutil" "os" "os/exec" "path" - "strings" "github.com/httprunner/hrp/internal/builtin" "github.com/httprunner/hrp/internal/ga" @@ -30,20 +28,20 @@ func CreateScaffold(projectName string) error { log.Info().Str("projectName", projectName).Msg("create new scaffold project") // create project folders - if err := createFolder(projectName); err != nil { + if err := builtin.CreateFolder(projectName); err != nil { return err } - if err := createFolder(path.Join(projectName, "har")); err != nil { + if err := builtin.CreateFolder(path.Join(projectName, "har")); err != nil { return err } - if err := createFolder(path.Join(projectName, "testcases")); err != nil { + if err := builtin.CreateFolder(path.Join(projectName, "testcases")); err != nil { return err } pluginDir := path.Join(projectName, "plugin") - if err := createFolder(pluginDir); err != nil { + if err := builtin.CreateFolder(pluginDir); err != nil { return err } - if err := createFolder(path.Join(projectName, "reports")); err != nil { + if err := builtin.CreateFolder(path.Join(projectName, "reports")); err != nil { return err } @@ -62,66 +60,33 @@ func CreateScaffold(projectName string) error { // create debugtalk.go pluginFile := path.Join(pluginDir, "debugtalk.go") - if err := createFile(pluginFile, demoPlugin); err != nil { + if err := builtin.CreateFile(pluginFile, demoPlugin); err != nil { return err } // create go mod - if err := execCommand(exec.Command("go", "mod", "init", "plugin"), pluginDir); err != nil { + if err := builtin.ExecCommand(exec.Command("go", "mod", "init", "plugin"), pluginDir); err != nil { return err } // download plugin dependency - if err := execCommand(exec.Command("go", "get", "github.com/httprunner/hrp/plugin"), pluginDir); err != nil { + if err := builtin.ExecCommand(exec.Command("go", "get", "github.com/httprunner/hrp/plugin"), pluginDir); err != nil { return err } // build plugin debugtalk.bin - if err := execCommand(exec.Command("go", "build", "-o", path.Join("..", "debugtalk.bin"), "debugtalk.go"), pluginDir); err != nil { + if err := builtin.ExecCommand(exec.Command("go", "build", "-o", path.Join("..", "debugtalk.bin"), "debugtalk.go"), pluginDir); err != nil { return err } // create .gitignore - if err := createFile(path.Join(projectName, ".gitignore"), demoIgnoreContent); err != nil { + if err := builtin.CreateFile(path.Join(projectName, ".gitignore"), demoIgnoreContent); err != nil { return err } // create .env - if err := createFile(path.Join(projectName, ".env"), demoEnvContent); err != nil { + if err := builtin.CreateFile(path.Join(projectName, ".env"), demoEnvContent); err != nil { return err } return nil } - -func execCommand(cmd *exec.Cmd, cwd string) error { - log.Info().Str("cmd", cmd.String()).Str("cwd", cwd).Msg("exec command") - cmd.Dir = cwd - output, err := cmd.CombinedOutput() - out := strings.TrimSpace(string(output)) - if err != nil { - log.Error().Err(err).Str("output", out).Msg("exec command failed") - } else if len(out) != 0 { - log.Info().Str("output", out).Msg("exec command success") - } - return err -} - -func createFolder(folderPath string) error { - log.Info().Str("path", folderPath).Msg("create folder") - err := os.MkdirAll(folderPath, os.ModePerm) - if err != nil { - log.Error().Err(err).Msg("create folder failed") - return err - } - return nil -} - -func createFile(filePath string, data string) error { - log.Info().Str("path", filePath).Msg("create file") - err := ioutil.WriteFile(filePath, []byte(data), 0o644) - if err != nil { - log.Error().Err(err).Msg("create file failed") - return err - } - return nil -} diff --git a/runner.go b/runner.go index b3830d48..8f2ca646 100644 --- a/runner.go +++ b/runner.go @@ -14,6 +14,7 @@ import ( "net/url" "os" "os/signal" + "path/filepath" "strconv" "strings" "sync" @@ -32,8 +33,8 @@ import ( ) const ( - summaryPath string = "summary.json" - reportPath string = "report.html" + summaryPath string = "reports/summary-%v.json" + reportPath string = "reports/report-%v.html" ) // Run starts to run API test with default configs. @@ -123,7 +124,8 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error { go ga.SendEvent(event) // report execution timing event defer ga.SendEvent(event.StartTiming("execution")) - + // record execution data to summary + s := newOutSummary() for _, iTestCase := range testcases { testcase, err := iTestCase.ToTestCase() if err != nil { @@ -137,7 +139,6 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error { log.Error().Interface("parameters", cfg.Parameters).Err(err).Msg("parse config parameters failed") return err } - s := newOutSummary() // 在runner模式下，指定整体策略，cfg.ParametersSetting.Iterators仅包含一个CartesianProduct的迭代器 for it := cfg.ParametersSetting.Iterators[0]; it.HasNext(); { // iterate through all parameter iterators and update case variables @@ -147,25 +148,32 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error { } } caseRunnerObj := r.newCaseRunner(testcase) - if err := caseRunnerObj.run(); err != nil { + if err = caseRunnerObj.run(); err != nil { log.Error().Err(err).Msg("[Run] run testcase failed") return err } caseSummary := caseRunnerObj.getSummary() s.appendCaseSummary(caseSummary) } - s.Time.Duration = time.Since(s.Time.StartAt).Seconds() - if r.saveTests { - err := builtin.Dump2JSON(s, summaryPath) - if err != nil { - return err - } + } + s.Time.Duration = time.Since(s.Time.StartAt).Seconds() + // save summary + if r.saveTests { + dir, _ := filepath.Split(summaryPath) + err := builtin.EnsureFolderExists(dir) + if err != nil { + return err } - if r.genHTMLReport { - err := genHTMLReport(s) - if err != nil { - return err - } + err = builtin.Dump2JSON(s, fmt.Sprintf(summaryPath, s.Time.StartAt.Unix())) + if err != nil { + return err + } + } + // generate HTML report + if r.genHTMLReport { + err := s.genHTMLReport() + if err != nil { + return err } } return nil @@ -232,14 +240,28 @@ func (r *caseRunner) run() error { Success: false, } } - r.summary.Records = append(r.summary.Records, stepDataObj) - r.summary.Success = r.summary.Success && stepDataObj.Success - r.summary.Stat.Total += 1 - if stepDataObj.Success { - r.summary.Stat.Successes += 1 - } else { - r.summary.Stat.Failures += 1 + if stepDataObj.StepType == stepTypeTestCase { + // merge test case if the step is test case + summary, ok := stepDataObj.Data.(*testCaseSummary) + if ok { + for _, rc := range summary.Records { + r.summary.Records = append(r.summary.Records, rc) + } + r.summary.Stat.Total += summary.Stat.Total + r.summary.Stat.Successes += summary.Stat.Successes + r.summary.Stat.Failures += summary.Stat.Failures + } + } else if stepDataObj.StepType == stepTypeRequest { + // only record that the test step is the request step + r.summary.Records = append(r.summary.Records, stepDataObj) + r.summary.Stat.Total += 1 + if stepDataObj.Success { + r.summary.Stat.Successes += 1 + } else { + r.summary.Stat.Failures += 1 + } } + r.summary.Success = r.summary.Success && stepDataObj.Success if err != nil { stepDataObj.Attachment = err.Error() if r.hrpRunner.failfast { @@ -853,8 +875,13 @@ func setBodyBytes(req *http.Request, data []byte) { //go:embed internal/report/template.html var reportTemplate string -func genHTMLReport(summary *Summary) error { - file, err := os.OpenFile(reportPath, os.O_WRONLY|os.O_CREATE, 0666) +func (s *Summary) genHTMLReport() error { + dir, _ := filepath.Split(reportPath) + err := builtin.EnsureFolderExists(dir) + if err != nil { + return err + } + file, err := os.OpenFile(fmt.Sprintf(reportPath, s.Time.StartAt.Unix()), os.O_WRONLY|os.O_CREATE, 0666) defer file.Close() if err != nil { log.Error().Err(err).Msg("open file failed") @@ -862,7 +889,7 @@ func genHTMLReport(summary *Summary) error { } writer := bufio.NewWriter(file) tmpl := template.Must(template.New("report").Parse(reportTemplate)) - err = tmpl.Execute(writer, summary) + err = tmpl.Execute(writer, s) if err != nil { log.Error().Err(err).Msg("execute applies a parsed template to the specified data object failed") return err diff --git a/runner_test.go b/runner_test.go index 2a7fce74..7e272825 100644 --- a/runner_test.go +++ b/runner_test.go @@ -42,7 +42,13 @@ func TestHttpRunner(t *testing.T) { Validate(). AssertEqual("status_code", 200, "check status code"). AssertEqual("headers.\"Content-Type\"", "application/json", "check http response Content-Type"), - NewStep("TestCase3").CallRefCase(&TestCase{Config: NewConfig("TestCase3")}), + NewStep("TestCase3").CallRefCase(&TestCase{Config: NewConfig("TestCase3").SetBaseURL("http://httpbin.org"), TestSteps: []IStep{ + NewStep("ip"). + GET("/ip"). + Validate(). + AssertEqual("status_code", 200, "check status code"). + AssertEqual("headers.\"Content-Type\"", "application/json", "check http response Content-Type"), + }}), }, } testcase2 := &TestCase{ @@ -50,7 +56,10 @@ func TestHttpRunner(t *testing.T) { } testcase3 := &TestCasePath{demoTestCaseJSONPath} - err := NewRunner(t).Run(testcase1, testcase2, testcase3) + r := NewRunner(t) + r.saveTests = true + r.genHTMLReport = true + err := r.Run(testcase1, testcase2, testcase3) if err != nil { t.Fatalf("run testcase error: %v", err) } @@ -136,7 +145,7 @@ func TestGenHTMLReport(t *testing.T) { caseSummary1.Records = []*stepData{stepResult1, stepResult2, nil} summary.appendCaseSummary(caseSummary1) summary.appendCaseSummary(caseSummary2) - err := genHTMLReport(summary) + err := summary.genHTMLReport() if err != nil { t.Error(err) } diff --git a/step_test.go b/step_test.go index 5eb6427f..b81b48d3 100644 --- a/step_test.go +++ b/step_test.go @@ -12,7 +12,6 @@ var ( WithCookies(map[string]string{"user": "debugtalk"}). Validate(). AssertEqual("status_code", 200, "check status code"). - AssertEqual("headers.Connection", "keep-alive", "check header Connection"). AssertEqual("headers.\"Content-Type\"", "application/json; charset=utf-8", "check header Content-Type"). AssertEqual("body.args.foo1", "bar1", "check param foo1"). AssertEqual("body.args.foo2", "bar2", "check param foo2")