Merge pull request #1212 from httprunner/refactor-run-folder

feat: support run testcases in specified folder path

.github/workflows/hrp-scaffold.yml

@@ -2,6 +2,8 @@ name: Run scaffold for hrp
 
 on:
   push:
+    branches:
+      - main
   pull_request:
     types: [synchronize]
 
@@ -26,7 +28,7 @@ jobs:
       - name: Run start project
         run: ./output/hrp startproject demo
       - name: Run generated demo tests
-        run: ./output/hrp run demo/testcases/demo_with_funplugin.json demo/testcases/demo_requests.yml demo/testcases/demo_ref_testcase.yml
+        run: ./output/hrp run demo/testcases/
       - name: Run demo in examples
         run: |
           ./output/hrp run examples/demo-with-py-plugin/testcases/demo_with_funplugin.json
@@ -51,7 +53,7 @@ jobs:
       - name: Run start project
         run: ./output/hrp startproject demo --go
       - name: Run generated demo tests
-        run: ./output/hrp run demo/testcases/demo_with_funplugin.json demo/testcases/demo_requests.yml demo/testcases/demo_ref_testcase.yml
+        run: ./output/hrp run demo/testcases/
       - name: Run demo in examples
         run: |
           go build -o examples/demo-with-go-plugin/debugtalk.bin examples/demo-with-go-plugin/plugin/debugtalk.go

.github/workflows/smoketest.yml

@@ -2,6 +2,8 @@ name: run smoke tests for httprunner
 
 on:
   push:
+    branches:
+      - main
   pull_request:
     types: [synchronize]
 

.github/workflows/unittest.yml

@@ -2,6 +2,8 @@ name: Run unittests
 
 on:
   push:
+    branches:
+      - main
   pull_request:
     types: [synchronize]
 

docs/CHANGELOG.md

@@ -7,6 +7,7 @@
 **go version**
 
 - feat: add `--profile` flag for har2case to support overwrite headers/cookies with specified yaml/json profile file
+- feat: support run testcases in specified folder path
 - change: integrate [sentry sdk][sentry sdk] for panic reporting and analysis
 - change: lock funplugin version when creating scaffold project
 - fix: call referenced api/testcase with relative path
@@ -30,8 +31,8 @@
 
 ## hrp-v0.7.0 (2022-03-15)
 
-- feat: support API layer for testcase #94
-- feat: support global headers for testcase #95
+- feat: support API layer for testcase
+- feat: support global headers for testcase
 - feat: support call referenced testcase by path in YAML/JSON testcases
 - fix: decode failure when content-encoding is deflate
 - fix: unstable RPS when load testing in high concurrency

hrp/boomer.go

@@ -38,11 +38,14 @@ func (b *HRPBoomer) Run(testcases ...ITestCase) {
 	defer sdk.SendEvent(event.StartTiming("execution"))
 
 	var taskSlice []*boomer.Task
-	for _, iTestCase := range testcases {
-		testcase, err := iTestCase.ToTestCase()
-		if err != nil {
-			panic(err)
-		}
+
+	// load all testcases
+	testCases, err := loadTestCases(testcases...)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, testcase := range testCases {
 		cfg := testcase.Config
 		err = initParameterIterator(cfg, "boomer")
 		if err != nil {

hrp/convert.go

@@ -90,74 +90,6 @@ func convertCheckExpr(checkExpr string) string {
 	return strings.Join(checkItems, ".")
 }
 
-func (tc *TCase) ToTestCase() (*TestCase, error) {
-	testCase := &TestCase{
-		Config: tc.Config,
-	}
-
-	// locate project root dir by plugin path
-	projectRootDir, err := getProjectRootDirPath(testCase.Config.Path)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get project root dir")
-	}
-	log.Info().Str("dir", projectRootDir).Msg("located project root dir")
-
-	for _, step := range tc.TestSteps {
-		if step.APIPath != "" {
-			path := filepath.Join(projectRootDir, step.APIPath)
-			if !builtin.IsFilePathExists(path) {
-				return nil, errors.New("referenced api file not found: " + path)
-			}
-
-			refAPI := APIPath(path)
-			step.APIContent = &refAPI
-			apiContent, err := step.APIContent.ToAPI()
-			if err != nil {
-				return nil, err
-			}
-			step.APIContent = apiContent
-			testCase.TestSteps = append(testCase.TestSteps, &StepAPIWithOptionalArgs{
-				step: step,
-			})
-		} else if step.TestCasePath != "" {
-			path := filepath.Join(projectRootDir, step.TestCasePath)
-			if !builtin.IsFilePathExists(path) {
-				return nil, errors.New("referenced testcase file not found: " + path)
-			}
-
-			refTestCase := TestCasePath(path)
-			step.TestCaseContent = &refTestCase
-			tc, err := step.TestCaseContent.ToTestCase()
-			if err != nil {
-				return nil, err
-			}
-			step.TestCaseContent = tc
-			testCase.TestSteps = append(testCase.TestSteps, &StepTestCaseWithOptionalArgs{
-				step: step,
-			})
-		} else if step.ThinkTime != nil {
-			testCase.TestSteps = append(testCase.TestSteps, &StepThinkTime{
-				step: step,
-			})
-		} else if step.Request != nil {
-			testCase.TestSteps = append(testCase.TestSteps, &StepRequestWithOptionalArgs{
-				step: step,
-			})
-		} else if step.Transaction != nil {
-			testCase.TestSteps = append(testCase.TestSteps, &StepTransaction{
-				step: step,
-			})
-		} else if step.Rendezvous != nil {
-			testCase.TestSteps = append(testCase.TestSteps, &StepRendezvous{
-				step: step,
-			})
-		} else {
-			log.Warn().Interface("step", step).Msg("[convertTestCase] unexpected step")
-		}
-	}
-	return testCase, nil
-}
-
 func getProjectRootDirPath(path string) (rootDir string, err error) {
 	pluginPath, err := locatePlugin(path)
 	if err == nil {
@@ -196,6 +128,7 @@ func (path *TestCasePath) GetPath() string {
 	return fmt.Sprintf("%v", *path)
 }
 
+// ToTestCase loads testcase path and convert to *TestCase
 func (path *TestCasePath) ToTestCase() (*TestCase, error) {
 	tc := &TCase{}
 	casePath := path.GetPath()
@@ -209,17 +142,76 @@ func (path *TestCasePath) ToTestCase() (*TestCase, error) {
 		return nil, err
 	}
 	tc.Config.Path = casePath
-	testcase, err := tc.ToTestCase()
-	if err != nil {
-		return nil, err
-	}
-	return testcase, nil
-}
 
-func (path *TestCasePath) ToTCase() (*TCase, error) {
-	testcase, err := path.ToTestCase()
-	if err != nil {
-		return nil, err
+	testCase := &TestCase{
+		Config: tc.Config,
 	}
-	return testcase.ToTCase()
+
+	// locate project root dir by plugin path
+	projectRootDir, err := getProjectRootDirPath(casePath)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get project root dir")
+	}
+
+	for _, step := range tc.TestSteps {
+		if step.API != nil {
+			apiPath, ok := step.API.(string)
+			if !ok {
+				return nil, fmt.Errorf("referenced api path should be string, got %v", step.API)
+			}
+			path := filepath.Join(projectRootDir, apiPath)
+			if !builtin.IsFilePathExists(path) {
+				return nil, errors.New("referenced api file not found: " + path)
+			}
+
+			refAPI := APIPath(path)
+			apiContent, err := refAPI.ToAPI()
+			if err != nil {
+				return nil, err
+			}
+			step.API = apiContent
+
+			testCase.TestSteps = append(testCase.TestSteps, &StepAPIWithOptionalArgs{
+				step: step,
+			})
+		} else if step.TestCase != nil {
+			casePath, ok := step.TestCase.(string)
+			if !ok {
+				return nil, fmt.Errorf("referenced testcase path should be string, got %v", step.TestCase)
+			}
+			path := filepath.Join(projectRootDir, casePath)
+			if !builtin.IsFilePathExists(path) {
+				return nil, errors.New("referenced testcase file not found: " + path)
+			}
+
+			refTestCase := TestCasePath(path)
+			tc, err := refTestCase.ToTestCase()
+			if err != nil {
+				return nil, err
+			}
+			step.TestCase = tc
+			testCase.TestSteps = append(testCase.TestSteps, &StepTestCaseWithOptionalArgs{
+				step: step,
+			})
+		} else if step.ThinkTime != nil {
+			testCase.TestSteps = append(testCase.TestSteps, &StepThinkTime{
+				step: step,
+			})
+		} else if step.Request != nil {
+			testCase.TestSteps = append(testCase.TestSteps, &StepRequestWithOptionalArgs{
+				step: step,
+			})
+		} else if step.Transaction != nil {
+			testCase.TestSteps = append(testCase.TestSteps, &StepTransaction{
+				step: step,
+			})
+		} else if step.Rendezvous != nil {
+			testCase.TestSteps = append(testCase.TestSteps, &StepRendezvous{
+				step: step,
+			})
+		} else {
+			log.Warn().Interface("step", step).Msg("[convertTestCase] unexpected step")
+		}
+	}
+	return testCase, nil
 }

hrp/convert_test.go

@@ -153,7 +153,7 @@ var demoTestCaseWithoutPlugin = &TestCase{
 }
 
 func TestGenDemoTestCase(t *testing.T) {
-	tCase, _ := demoTestCaseWithPlugin.ToTCase()
+	tCase := demoTestCaseWithPlugin.ToTCase()
 	err := builtin.Dump2JSON(tCase, demoTestCaseWithPluginJSONPath.GetPath())
 	if err != nil {
 		t.Fail()
@@ -163,7 +163,7 @@ func TestGenDemoTestCase(t *testing.T) {
 		t.Fail()
 	}
 
-	tCase, _ = demoTestCaseWithoutPlugin.ToTCase()
+	tCase = demoTestCaseWithoutPlugin.ToTCase()
 	err = builtin.Dump2JSON(tCase, demoTestCaseWithoutPluginJSONPath.GetPath())
 	if err != nil {
 		t.Fail()

hrp/internal/builtin/utils.go

@@ -132,6 +132,18 @@ func IsFilePathExists(path string) bool {
 	return true
 }
 
+// IsFolderPathExists returns true if path exists and path is folder
+func IsFolderPathExists(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil {
+		// path not exists
+		return false
+	}
+
+	// path exists and is dir
+	return info.IsDir()
+}
+
 func EnsureFolderExists(folderPath string) error {
 	if !IsPathExists(folderPath) {
 		err := CreateFolder(folderPath)

hrp/models.go

@@ -222,21 +222,19 @@ type IAPI interface {
 // TStep represents teststep data structure.
 // Each step maybe two different type: make one HTTP request or reference another testcase.
 type TStep struct {
-	Name            string                 `json:"name" yaml:"name"` // required
-	Request         *Request               `json:"request,omitempty" yaml:"request,omitempty"`
-	APIPath         string                 `json:"api,omitempty" yaml:"api,omitempty"`
-	APIContent      IAPI                   `json:"api_content,omitempty" yaml:"api_content,omitempty"`
-	TestCasePath    string                 `json:"testcase,omitempty" yaml:"testcase,omitempty"`
-	TestCaseContent ITestCase              `json:"testcase_content,omitempty" yaml:"testcase_content,omitempty"`
-	Transaction     *Transaction           `json:"transaction,omitempty" yaml:"transaction,omitempty"`
-	Rendezvous      *Rendezvous            `json:"rendezvous,omitempty" yaml:"rendezvous,omitempty"`
-	ThinkTime       *ThinkTime             `json:"think_time,omitempty" yaml:"think_time,omitempty"`
-	Variables       map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"`
-	SetupHooks      []string               `json:"setup_hooks,omitempty" yaml:"setup_hooks,omitempty"`
-	TeardownHooks   []string               `json:"teardown_hooks,omitempty" yaml:"teardown_hooks,omitempty"`
-	Extract         map[string]string      `json:"extract,omitempty" yaml:"extract,omitempty"`
-	Validators      []interface{}          `json:"validate,omitempty" yaml:"validate,omitempty"`
-	Export          []string               `json:"export,omitempty" yaml:"export,omitempty"`
+	Name          string                 `json:"name" yaml:"name"` // required
+	Request       *Request               `json:"request,omitempty" yaml:"request,omitempty"`
+	API           interface{}            `json:"api,omitempty" yaml:"api,omitempty"`           // *APIPath or *API
+	TestCase      interface{}            `json:"testcase,omitempty" yaml:"testcase,omitempty"` // *TestCasePath or *TestCase
+	Transaction   *Transaction           `json:"transaction,omitempty" yaml:"transaction,omitempty"`
+	Rendezvous    *Rendezvous            `json:"rendezvous,omitempty" yaml:"rendezvous,omitempty"`
+	ThinkTime     *ThinkTime             `json:"think_time,omitempty" yaml:"think_time,omitempty"`
+	Variables     map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"`
+	SetupHooks    []string               `json:"setup_hooks,omitempty" yaml:"setup_hooks,omitempty"`
+	TeardownHooks []string               `json:"teardown_hooks,omitempty" yaml:"teardown_hooks,omitempty"`
+	Extract       map[string]string      `json:"extract,omitempty" yaml:"extract,omitempty"`
+	Validators    []interface{}          `json:"validate,omitempty" yaml:"validate,omitempty"`
+	Export        []string               `json:"export,omitempty" yaml:"export,omitempty"`
 }
 
 type stepType string
@@ -293,10 +291,6 @@ type TCase struct {
 	TestSteps []*TStep `json:"teststeps" yaml:"teststeps"`
 }
 
-func (tc *TCase) Path() string {
-	return tc.Config.Path
-}
-
 // IStep represents interface for all types for teststeps, includes:
 // StepRequest, StepRequestWithOptionalArgs, StepRequestValidation, StepRequestExtraction,
 // StepTestCaseWithOptionalArgs,
@@ -312,7 +306,6 @@ type IStep interface {
 type ITestCase interface {
 	GetPath() string
 	ToTestCase() (*TestCase, error)
-	ToTCase() (*TCase, error)
 }
 
 // TestCase is a container for one testcase, which is used for testcase runner.
@@ -330,14 +323,14 @@ func (tc *TestCase) ToTestCase() (*TestCase, error) {
 	return tc, nil
 }
 
-func (tc *TestCase) ToTCase() (*TCase, error) {
-	tCase := TCase{
+func (tc *TestCase) ToTCase() *TCase {
+	tCase := &TCase{
 		Config: tc.Config,
 	}
 	for _, step := range tc.TestSteps {
 		tCase.TestSteps = append(tCase.TestSteps, step.ToStruct())
 	}
-	return &tCase, nil
+	return tCase
 }
 
 type testCaseStat struct {

hrp/runner.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"html/template"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -148,15 +149,17 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error {
 	defer sdk.SendEvent(event.StartTiming("execution"))
 	// record execution data to summary
 	s := newOutSummary()
-	for _, iTestCase := range testcases {
-		testcase, err := iTestCase.ToTestCase()
-		if err != nil {
-			log.Error().Err(err).Msg("[Run] convert ITestCase interface to TestCase struct failed")
-			return err
-		}
+
+	// load all testcases
+	testCases, err := loadTestCases(testcases...)
+	if err != nil {
+		return err
+	}
+
+	for _, testcase := range testCases {
 		cfg := testcase.Config
 		// parse config parameters
-		err = initParameterIterator(cfg, "runner")
+		err := initParameterIterator(cfg, "runner")
 		if err != nil {
 			log.Error().Interface("parameters", cfg.Parameters).Err(err).Msg("parse config parameters failed")
 			return err
@@ -201,6 +204,62 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error {
 	return nil
 }
 
+func loadTestCases(iTestCases ...ITestCase) ([]*TestCase, error) {
+	testCases := make([]*TestCase, 0)
+
+	for _, iTestCase := range iTestCases {
+		if _, ok := iTestCase.(*TestCase); ok {
+			testcase, err := iTestCase.ToTestCase()
+			if err != nil {
+				log.Error().Err(err).Msg("failed to convert ITestCase interface to TestCase struct")
+				return nil, err
+			}
+			testCases = append(testCases, testcase)
+			continue
+		}
+
+		// iTestCase should be a TestCasePath, file path or folder path
+		testCasePath, ok := iTestCase.(*TestCasePath)
+		if !ok {
+			return nil, errors.New("invalid iTestCase type")
+		}
+
+		casePaths := make([]*TestCasePath, 0)
+		casePath := iTestCase.GetPath()
+		if builtin.IsFolderPathExists(casePath) {
+			// folder path
+			files, err := ioutil.ReadDir(casePath)
+			if err != nil {
+				return nil, errors.Wrap(err, "read dir failed")
+			}
+			for _, f := range files {
+				ext := filepath.Ext(f.Name())
+				if ext != ".yml" && ext != ".yaml" && ext != ".json" {
+					// ignore non-testcase files
+					continue
+				}
+				path := TestCasePath(filepath.Join(casePath, f.Name()))
+				casePaths = append(casePaths, &path)
+			}
+		} else {
+			// file path
+			casePaths = append(casePaths, testCasePath)
+		}
+
+		for _, path := range casePaths {
+			tc, err := path.ToTestCase()
+			if err != nil {
+				log.Error().Err(err).Str("path", path.GetPath()).Msg("load testcase failed")
+				return nil, errors.Wrap(err, "load testcase failed")
+			}
+			testCases = append(testCases, tc)
+		}
+	}
+
+	log.Info().Int("count", len(testCases)).Msg("load testcases successfully")
+	return testCases, nil
+}
+
 func (r *HRPRunner) newCaseRunner(testcase *TestCase) *caseRunner {
 	caseRunner := &caseRunner{
 		TestCase:  testcase,
@@ -345,7 +404,7 @@ func (r *caseRunner) runStep(index int, caseConfig *TConfig) (stepResult *stepDa
 		if _, ok := step.(*StepAPIWithOptionalArgs); ok {
 			// run referenced API
 			log.Info().Str("api", copiedStep.Name).Msg("run referenced api")
-			api, _ := copiedStep.APIContent.ToAPI()
+			api, _ := copiedStep.API.(*API)
 			extendWithAPI(copiedStep, api)
 		}
 		// override headers
@@ -508,7 +567,7 @@ func (r *caseRunner) runStepRendezvous(rendezvous *Rendezvous) (stepResult *step
 }
 
 func (r *caseRunner) isPreRendezvousAllReleased(rendezvous *Rendezvous) bool {
-	tCase, _ := r.ToTCase()
+	tCase := r.TestCase.ToTCase()
 	for _, step := range tCase.TestSteps {
 		preRendezvous := step.Rendezvous
 		if preRendezvous == nil {
@@ -555,7 +614,7 @@ func (r *Rendezvous) setReleased() {
 }
 
 func initRendezvous(testcase *TestCase, total int64) []*Rendezvous {
-	tCase, _ := testcase.ToTCase()
+	tCase := testcase.ToTCase()
 	var rendezvousList []*Rendezvous
 	for _, step := range tCase.TestSteps {
 		if step.Rendezvous == nil {
@@ -967,7 +1026,7 @@ func (r *caseRunner) runStepTestCase(step *TStep) (stepResult *stepData, err err
 		StepType: stepTypeTestCase,
 		Success:  false,
 	}
-	testcase := step.TestCaseContent
+	testcase := step.TestCase.(*TestCase)
 
 	// copy testcase to avoid data racing
 	copiedTestCase := &TestCase{}

hrp/runner_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/rs/zerolog/log"
+	"github.com/stretchr/testify/assert"
 
 	"github.com/httprunner/httprunner/hrp/internal/scaffold"
 )
@@ -283,3 +284,37 @@ func TestRunCaseWithRefAPI(t *testing.T) {
 		t.Fail()
 	}
 }
+
+func TestLoadTestCases(t *testing.T) {
+	// load test cases from folder path
+	tc := TestCasePath("../examples/demo-with-py-plugin/testcases/")
+	testCases, err := loadTestCases(&tc)
+	if !assert.Nil(t, err) {
+		t.Fail()
+	}
+	if !assert.Equal(t, len(testCases), 3) {
+		t.Fail()
+	}
+
+	// load test cases from single file path
+	tc = demoTestCaseWithPluginJSONPath
+	testCases, err = loadTestCases(&tc)
+	if !assert.Nil(t, err) {
+		t.Fail()
+	}
+	if !assert.Equal(t, len(testCases), 1) {
+		t.Fail()
+	}
+
+	// load test cases from TestCase instance
+	testcase := &TestCase{
+		Config: NewConfig("TestCase").SetWeight(3),
+	}
+	testCases, err = loadTestCases(testcase)
+	if !assert.Nil(t, err) {
+		t.Fail()
+	}
+	if !assert.Equal(t, len(testCases), 1) {
+		t.Fail()
+	}
+}

hrp/step.go

@@ -169,7 +169,7 @@ func (s *StepRequest) PATCH(url string) *StepRequestWithOptionalArgs {
 // CallRefCase calls a referenced testcase.
 func (s *StepRequest) CallRefCase(tc ITestCase) *StepTestCaseWithOptionalArgs {
 	var err error
-	s.step.TestCaseContent, err = tc.ToTestCase()
+	s.step.TestCase, err = tc.ToTestCase()
 	if err != nil {
 		log.Error().Err(err).Msg("failed to load testcase")
 		os.Exit(1)
@@ -182,7 +182,7 @@ func (s *StepRequest) CallRefCase(tc ITestCase) *StepTestCaseWithOptionalArgs {
 // CallRefAPI calls a referenced api.
 func (s *StepRequest) CallRefAPI(api IAPI) *StepAPIWithOptionalArgs {
 	var err error
-	s.step.APIContent, err = api.ToAPI()
+	s.step.API, err = api.ToAPI()
 	if err != nil {
 		log.Error().Err(err).Msg("failed to load api")
 		os.Exit(1)
@@ -332,8 +332,10 @@ func (s *StepAPIWithOptionalArgs) TeardownHook(hook string) *StepAPIWithOptional
 
 // Export specifies variable names to export from referenced api for current step.
 func (s *StepAPIWithOptionalArgs) Export(names ...string) *StepAPIWithOptionalArgs {
-	api, _ := s.step.APIContent.ToAPI()
-	s.step.Export = append(api.Export, names...)
+	api, ok := s.step.API.(*API)
+	if ok {
+		s.step.Export = append(api.Export, names...)
+	}
 	return s
 }
 
@@ -341,8 +343,11 @@ func (s *StepAPIWithOptionalArgs) Name() string {
 	if s.step.Name != "" {
 		return s.step.Name
 	}
-	api, _ := s.step.APIContent.ToAPI()
-	return api.Name
+	api, ok := s.step.API.(*API)
+	if ok {
+		return api.Name
+	}
+	return ""
 }
 
 func (s *StepAPIWithOptionalArgs) Type() string {
@@ -374,8 +379,11 @@ func (s *StepTestCaseWithOptionalArgs) Name() string {
 	if s.step.Name != "" {
 		return s.step.Name
 	}
-	ts, _ := s.step.TestCaseContent.ToTestCase()
-	return ts.Config.Name
+	ts, ok := s.step.TestCase.(*TestCase)
+	if ok {
+		return ts.Config.Name
+	}
+	return ""
 }
 
 func (s *StepTestCaseWithOptionalArgs) Type() string {