Merge pull request #85 from xucong053/main

change: merge teststeps when call other testcases
change: add timestamp for generated summary and html reports

internal/builtin/function.go

@@ -6,9 +6,12 @@ import (
 	"encoding/csv"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"math"
 	"math/rand"
+	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -142,3 +145,70 @@ func FormatResponse(raw interface{}) interface{} {
 	}
 	return formattedResponse
 }
+
+func ExecCommand(cmd *exec.Cmd, cwd string) error {
+	log.Info().Str("cmd", cmd.String()).Str("cwd", cwd).Msg("exec command")
+	cmd.Dir = cwd
+	output, err := cmd.CombinedOutput()
+	out := strings.TrimSpace(string(output))
+	if err != nil {
+		log.Error().Err(err).Str("output", out).Msg("exec command failed")
+	} else if len(out) != 0 {
+		log.Info().Str("output", out).Msg("exec command success")
+	}
+	return err
+}
+
+func CreateFolder(folderPath string) error {
+	log.Info().Str("path", folderPath).Msg("create folder")
+	err := os.MkdirAll(folderPath, os.ModePerm)
+	if err != nil {
+		log.Error().Err(err).Msg("create folder failed")
+		return err
+	}
+	return nil
+}
+
+func CreateFile(filePath string, data string) error {
+	log.Info().Str("path", filePath).Msg("create file")
+	err := ioutil.WriteFile(filePath, []byte(data), 0o644)
+	if err != nil {
+		log.Error().Err(err).Msg("create file failed")
+		return err
+	}
+	return nil
+}
+
+// isFilePathExists returns true if path exists, whether path is file or dir
+func isPathExists(path string) bool {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+// isFilePathExists returns true if path exists and path is file
+func isFilePathExists(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil {
+		// path not exists
+		return false
+	}
+
+	// path exists
+	if info.IsDir() {
+		// path is dir, not file
+		return false
+	}
+	return true
+}
+
+func EnsureFolderExists(folderPath string) error {
+	if !isPathExists(folderPath) {
+		err := CreateFolder(folderPath)
+		return err
+	} else if isFilePathExists(folderPath) {
+		return fmt.Errorf("path %v should be directory", folderPath)
+	}
+	return nil
+}

internal/scaffold/main.go

@@ -2,11 +2,9 @@ package scaffold
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
-	"strings"
 
 	"github.com/httprunner/hrp/internal/builtin"
 	"github.com/httprunner/hrp/internal/ga"
@@ -30,20 +28,20 @@ func CreateScaffold(projectName string) error {
 	log.Info().Str("projectName", projectName).Msg("create new scaffold project")
 
 	// create project folders
-	if err := createFolder(projectName); err != nil {
+	if err := builtin.CreateFolder(projectName); err != nil {
 		return err
 	}
-	if err := createFolder(path.Join(projectName, "har")); err != nil {
+	if err := builtin.CreateFolder(path.Join(projectName, "har")); err != nil {
 		return err
 	}
-	if err := createFolder(path.Join(projectName, "testcases")); err != nil {
+	if err := builtin.CreateFolder(path.Join(projectName, "testcases")); err != nil {
 		return err
 	}
 	pluginDir := path.Join(projectName, "plugin")
-	if err := createFolder(pluginDir); err != nil {
+	if err := builtin.CreateFolder(pluginDir); err != nil {
 		return err
 	}
-	if err := createFolder(path.Join(projectName, "reports")); err != nil {
+	if err := builtin.CreateFolder(path.Join(projectName, "reports")); err != nil {
 		return err
 	}
 
@@ -62,66 +60,33 @@ func CreateScaffold(projectName string) error {
 
 	// create debugtalk.go
 	pluginFile := path.Join(pluginDir, "debugtalk.go")
-	if err := createFile(pluginFile, demoPlugin); err != nil {
+	if err := builtin.CreateFile(pluginFile, demoPlugin); err != nil {
 		return err
 	}
 
 	// create go mod
-	if err := execCommand(exec.Command("go", "mod", "init", "plugin"), pluginDir); err != nil {
+	if err := builtin.ExecCommand(exec.Command("go", "mod", "init", "plugin"), pluginDir); err != nil {
 		return err
 	}
 
 	// download plugin dependency
-	if err := execCommand(exec.Command("go", "get", "github.com/httprunner/hrp/plugin"), pluginDir); err != nil {
+	if err := builtin.ExecCommand(exec.Command("go", "get", "github.com/httprunner/hrp/plugin"), pluginDir); err != nil {
 		return err
 	}
 
 	// build plugin debugtalk.bin
-	if err := execCommand(exec.Command("go", "build", "-o", path.Join("..", "debugtalk.bin"), "debugtalk.go"), pluginDir); err != nil {
+	if err := builtin.ExecCommand(exec.Command("go", "build", "-o", path.Join("..", "debugtalk.bin"), "debugtalk.go"), pluginDir); err != nil {
 		return err
 	}
 
 	// create .gitignore
-	if err := createFile(path.Join(projectName, ".gitignore"), demoIgnoreContent); err != nil {
+	if err := builtin.CreateFile(path.Join(projectName, ".gitignore"), demoIgnoreContent); err != nil {
 		return err
 	}
 	// create .env
-	if err := createFile(path.Join(projectName, ".env"), demoEnvContent); err != nil {
+	if err := builtin.CreateFile(path.Join(projectName, ".env"), demoEnvContent); err != nil {
 		return err
 	}
 
 	return nil
 }
-
-func execCommand(cmd *exec.Cmd, cwd string) error {
-	log.Info().Str("cmd", cmd.String()).Str("cwd", cwd).Msg("exec command")
-	cmd.Dir = cwd
-	output, err := cmd.CombinedOutput()
-	out := strings.TrimSpace(string(output))
-	if err != nil {
-		log.Error().Err(err).Str("output", out).Msg("exec command failed")
-	} else if len(out) != 0 {
-		log.Info().Str("output", out).Msg("exec command success")
-	}
-	return err
-}
-
-func createFolder(folderPath string) error {
-	log.Info().Str("path", folderPath).Msg("create folder")
-	err := os.MkdirAll(folderPath, os.ModePerm)
-	if err != nil {
-		log.Error().Err(err).Msg("create folder failed")
-		return err
-	}
-	return nil
-}
-
-func createFile(filePath string, data string) error {
-	log.Info().Str("path", filePath).Msg("create file")
-	err := ioutil.WriteFile(filePath, []byte(data), 0o644)
-	if err != nil {
-		log.Error().Err(err).Msg("create file failed")
-		return err
-	}
-	return nil
-}

runner.go

@@ -14,6 +14,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -32,8 +33,8 @@ import (
 )
 
 const (
-	summaryPath string = "summary.json"
-	reportPath  string = "report.html"
+	summaryPath string = "reports/summary-%v.json"
+	reportPath  string = "reports/report-%v.html"
 )
 
 // Run starts to run API test with default configs.
@@ -123,7 +124,8 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error {
 	go ga.SendEvent(event)
 	// report execution timing event
 	defer ga.SendEvent(event.StartTiming("execution"))
-
+	// record execution data to summary
+	s := newOutSummary()
 	for _, iTestCase := range testcases {
 		testcase, err := iTestCase.ToTestCase()
 		if err != nil {
@@ -137,7 +139,6 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error {
 			log.Error().Interface("parameters", cfg.Parameters).Err(err).Msg("parse config parameters failed")
 			return err
 		}
-		s := newOutSummary()
 		// 在runner模式下，指定整体策略，cfg.ParametersSetting.Iterators仅包含一个CartesianProduct的迭代器
 		for it := cfg.ParametersSetting.Iterators[0]; it.HasNext(); {
 			// iterate through all parameter iterators and update case variables
@@ -147,25 +148,32 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error {
 				}
 			}
 			caseRunnerObj := r.newCaseRunner(testcase)
-			if err := caseRunnerObj.run(); err != nil {
+			if err = caseRunnerObj.run(); err != nil {
 				log.Error().Err(err).Msg("[Run] run testcase failed")
 				return err
 			}
 			caseSummary := caseRunnerObj.getSummary()
 			s.appendCaseSummary(caseSummary)
 		}
-		s.Time.Duration = time.Since(s.Time.StartAt).Seconds()
-		if r.saveTests {
-			err := builtin.Dump2JSON(s, summaryPath)
-			if err != nil {
-				return err
-			}
+	}
+	s.Time.Duration = time.Since(s.Time.StartAt).Seconds()
+	// save summary
+	if r.saveTests {
+		dir, _ := filepath.Split(summaryPath)
+		err := builtin.EnsureFolderExists(dir)
+		if err != nil {
+			return err
 		}
-		if r.genHTMLReport {
-			err := genHTMLReport(s)
-			if err != nil {
-				return err
-			}
+		err = builtin.Dump2JSON(s, fmt.Sprintf(summaryPath, s.Time.StartAt.Unix()))
+		if err != nil {
+			return err
+		}
+	}
+	// generate HTML report
+	if r.genHTMLReport {
+		err := s.genHTMLReport()
+		if err != nil {
+			return err
 		}
 	}
 	return nil
@@ -232,14 +240,28 @@ func (r *caseRunner) run() error {
 				Success: false,
 			}
 		}
-		r.summary.Records = append(r.summary.Records, stepDataObj)
-		r.summary.Success = r.summary.Success && stepDataObj.Success
-		r.summary.Stat.Total += 1
-		if stepDataObj.Success {
-			r.summary.Stat.Successes += 1
-		} else {
-			r.summary.Stat.Failures += 1
+		if stepDataObj.StepType == stepTypeTestCase {
+			// merge test case if the step is test case
+			summary, ok := stepDataObj.Data.(*testCaseSummary)
+			if ok {
+				for _, rc := range summary.Records {
+					r.summary.Records = append(r.summary.Records, rc)
+				}
+				r.summary.Stat.Total += summary.Stat.Total
+				r.summary.Stat.Successes += summary.Stat.Successes
+				r.summary.Stat.Failures += summary.Stat.Failures
+			}
+		} else if stepDataObj.StepType == stepTypeRequest {
+			// only record that the test step is the request step
+			r.summary.Records = append(r.summary.Records, stepDataObj)
+			r.summary.Stat.Total += 1
+			if stepDataObj.Success {
+				r.summary.Stat.Successes += 1
+			} else {
+				r.summary.Stat.Failures += 1
+			}
 		}
+		r.summary.Success = r.summary.Success && stepDataObj.Success
 		if err != nil {
 			stepDataObj.Attachment = err.Error()
 			if r.hrpRunner.failfast {
@@ -853,8 +875,13 @@ func setBodyBytes(req *http.Request, data []byte) {
 //go:embed internal/report/template.html
 var reportTemplate string
 
-func genHTMLReport(summary *Summary) error {
-	file, err := os.OpenFile(reportPath, os.O_WRONLY|os.O_CREATE, 0666)
+func (s *Summary) genHTMLReport() error {
+	dir, _ := filepath.Split(reportPath)
+	err := builtin.EnsureFolderExists(dir)
+	if err != nil {
+		return err
+	}
+	file, err := os.OpenFile(fmt.Sprintf(reportPath, s.Time.StartAt.Unix()), os.O_WRONLY|os.O_CREATE, 0666)
 	defer file.Close()
 	if err != nil {
 		log.Error().Err(err).Msg("open file failed")
@@ -862,7 +889,7 @@ func genHTMLReport(summary *Summary) error {
 	}
 	writer := bufio.NewWriter(file)
 	tmpl := template.Must(template.New("report").Parse(reportTemplate))
-	err = tmpl.Execute(writer, summary)
+	err = tmpl.Execute(writer, s)
 	if err != nil {
 		log.Error().Err(err).Msg("execute applies a parsed template to the specified data object failed")
 		return err

runner_test.go

@@ -42,7 +42,13 @@ func TestHttpRunner(t *testing.T) {
 				Validate().
 				AssertEqual("status_code", 200, "check status code").
 				AssertEqual("headers.\"Content-Type\"", "application/json", "check http response Content-Type"),
-			NewStep("TestCase3").CallRefCase(&TestCase{Config: NewConfig("TestCase3")}),
+			NewStep("TestCase3").CallRefCase(&TestCase{Config: NewConfig("TestCase3").SetBaseURL("http://httpbin.org"), TestSteps: []IStep{
+				NewStep("ip").
+					GET("/ip").
+					Validate().
+					AssertEqual("status_code", 200, "check status code").
+					AssertEqual("headers.\"Content-Type\"", "application/json", "check http response Content-Type"),
+			}}),
 		},
 	}
 	testcase2 := &TestCase{
@@ -50,7 +56,10 @@ func TestHttpRunner(t *testing.T) {
 	}
 	testcase3 := &TestCasePath{demoTestCaseJSONPath}
 
-	err := NewRunner(t).Run(testcase1, testcase2, testcase3)
+	r := NewRunner(t)
+	r.saveTests = true
+	r.genHTMLReport = true
+	err := r.Run(testcase1, testcase2, testcase3)
 	if err != nil {
 		t.Fatalf("run testcase error: %v", err)
 	}
@@ -136,7 +145,7 @@ func TestGenHTMLReport(t *testing.T) {
 	caseSummary1.Records = []*stepData{stepResult1, stepResult2, nil}
 	summary.appendCaseSummary(caseSummary1)
 	summary.appendCaseSummary(caseSummary2)
-	err := genHTMLReport(summary)
+	err := summary.genHTMLReport()
 	if err != nil {
 		t.Error(err)
 	}