diff --git a/.github/workflows/hrp-scaffold.yml b/.github/workflows/hrp-scaffold.yml index 0bf56c20..b5caed98 100644 --- a/.github/workflows/hrp-scaffold.yml +++ b/.github/workflows/hrp-scaffold.yml @@ -2,6 +2,8 @@ name: Run scaffold for hrp on: push: + branches: + - main pull_request: types: [synchronize] @@ -26,7 +28,7 @@ jobs: - name: Run start project run: ./output/hrp startproject demo - name: Run generated demo tests - run: ./output/hrp run demo/testcases/demo_with_funplugin.json demo/testcases/demo_requests.yml demo/testcases/demo_ref_testcase.yml + run: ./output/hrp run demo/testcases/ - name: Run demo in examples run: | ./output/hrp run examples/demo-with-py-plugin/testcases/demo_with_funplugin.json @@ -51,7 +53,7 @@ jobs: - name: Run start project run: ./output/hrp startproject demo --go - name: Run generated demo tests - run: ./output/hrp run demo/testcases/demo_with_funplugin.json demo/testcases/demo_requests.yml demo/testcases/demo_ref_testcase.yml + run: ./output/hrp run demo/testcases/ - name: Run demo in examples run: | go build -o examples/demo-with-go-plugin/debugtalk.bin examples/demo-with-go-plugin/plugin/debugtalk.go diff --git a/.github/workflows/smoketest.yml b/.github/workflows/smoketest.yml index a638edfd..90bf1e8e 100644 --- a/.github/workflows/smoketest.yml +++ b/.github/workflows/smoketest.yml @@ -2,6 +2,8 @@ name: run smoke tests for httprunner on: push: + branches: + - main pull_request: types: [synchronize] diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index fb6e3b5d..7ee267d3 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -2,6 +2,8 @@ name: Run unittests on: push: + branches: + - main pull_request: types: [synchronize] diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c726e78b..3c7931d9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,7 @@ **go version** - feat: add `--profile` flag for har2case to support overwrite headers/cookies with specified yaml/json profile file +- feat: support run testcases in specified folder path - change: integrate [sentry sdk][sentry sdk] for panic reporting and analysis - change: lock funplugin version when creating scaffold project - fix: call referenced api/testcase with relative path @@ -30,8 +31,8 @@ ## hrp-v0.7.0 (2022-03-15) -- feat: support API layer for testcase #94 -- feat: support global headers for testcase #95 +- feat: support API layer for testcase +- feat: support global headers for testcase - feat: support call referenced testcase by path in YAML/JSON testcases - fix: decode failure when content-encoding is deflate - fix: unstable RPS when load testing in high concurrency diff --git a/hrp/boomer.go b/hrp/boomer.go index 9d0cf667..8d49296f 100644 --- a/hrp/boomer.go +++ b/hrp/boomer.go @@ -38,11 +38,14 @@ func (b *HRPBoomer) Run(testcases ...ITestCase) { defer sdk.SendEvent(event.StartTiming("execution")) var taskSlice []*boomer.Task - for _, iTestCase := range testcases { - testcase, err := iTestCase.ToTestCase() - if err != nil { - panic(err) - } + + // load all testcases + testCases, err := loadTestCases(testcases...) + if err != nil { + panic(err) + } + + for _, testcase := range testCases { cfg := testcase.Config err = initParameterIterator(cfg, "boomer") if err != nil { diff --git a/hrp/convert.go b/hrp/convert.go index 3148ab56..035bfadc 100644 --- a/hrp/convert.go +++ b/hrp/convert.go @@ -90,74 +90,6 @@ func convertCheckExpr(checkExpr string) string { return strings.Join(checkItems, ".") } -func (tc *TCase) ToTestCase() (*TestCase, error) { - testCase := &TestCase{ - Config: tc.Config, - } - - // locate project root dir by plugin path - projectRootDir, err := getProjectRootDirPath(testCase.Config.Path) - if err != nil { - return nil, errors.Wrap(err, "failed to get project root dir") - } - log.Info().Str("dir", projectRootDir).Msg("located project root dir") - - for _, step := range tc.TestSteps { - if step.APIPath != "" { - path := filepath.Join(projectRootDir, step.APIPath) - if !builtin.IsFilePathExists(path) { - return nil, errors.New("referenced api file not found: " + path) - } - - refAPI := APIPath(path) - step.APIContent = &refAPI - apiContent, err := step.APIContent.ToAPI() - if err != nil { - return nil, err - } - step.APIContent = apiContent - testCase.TestSteps = append(testCase.TestSteps, &StepAPIWithOptionalArgs{ - step: step, - }) - } else if step.TestCasePath != "" { - path := filepath.Join(projectRootDir, step.TestCasePath) - if !builtin.IsFilePathExists(path) { - return nil, errors.New("referenced testcase file not found: " + path) - } - - refTestCase := TestCasePath(path) - step.TestCaseContent = &refTestCase - tc, err := step.TestCaseContent.ToTestCase() - if err != nil { - return nil, err - } - step.TestCaseContent = tc - testCase.TestSteps = append(testCase.TestSteps, &StepTestCaseWithOptionalArgs{ - step: step, - }) - } else if step.ThinkTime != nil { - testCase.TestSteps = append(testCase.TestSteps, &StepThinkTime{ - step: step, - }) - } else if step.Request != nil { - testCase.TestSteps = append(testCase.TestSteps, &StepRequestWithOptionalArgs{ - step: step, - }) - } else if step.Transaction != nil { - testCase.TestSteps = append(testCase.TestSteps, &StepTransaction{ - step: step, - }) - } else if step.Rendezvous != nil { - testCase.TestSteps = append(testCase.TestSteps, &StepRendezvous{ - step: step, - }) - } else { - log.Warn().Interface("step", step).Msg("[convertTestCase] unexpected step") - } - } - return testCase, nil -} - func getProjectRootDirPath(path string) (rootDir string, err error) { pluginPath, err := locatePlugin(path) if err == nil { @@ -196,6 +128,7 @@ func (path *TestCasePath) GetPath() string { return fmt.Sprintf("%v", *path) } +// ToTestCase loads testcase path and convert to *TestCase func (path *TestCasePath) ToTestCase() (*TestCase, error) { tc := &TCase{} casePath := path.GetPath() @@ -209,17 +142,76 @@ func (path *TestCasePath) ToTestCase() (*TestCase, error) { return nil, err } tc.Config.Path = casePath - testcase, err := tc.ToTestCase() - if err != nil { - return nil, err - } - return testcase, nil -} -func (path *TestCasePath) ToTCase() (*TCase, error) { - testcase, err := path.ToTestCase() - if err != nil { - return nil, err + testCase := &TestCase{ + Config: tc.Config, } - return testcase.ToTCase() + + // locate project root dir by plugin path + projectRootDir, err := getProjectRootDirPath(casePath) + if err != nil { + return nil, errors.Wrap(err, "failed to get project root dir") + } + + for _, step := range tc.TestSteps { + if step.API != nil { + apiPath, ok := step.API.(string) + if !ok { + return nil, fmt.Errorf("referenced api path should be string, got %v", step.API) + } + path := filepath.Join(projectRootDir, apiPath) + if !builtin.IsFilePathExists(path) { + return nil, errors.New("referenced api file not found: " + path) + } + + refAPI := APIPath(path) + apiContent, err := refAPI.ToAPI() + if err != nil { + return nil, err + } + step.API = apiContent + + testCase.TestSteps = append(testCase.TestSteps, &StepAPIWithOptionalArgs{ + step: step, + }) + } else if step.TestCase != nil { + casePath, ok := step.TestCase.(string) + if !ok { + return nil, fmt.Errorf("referenced testcase path should be string, got %v", step.TestCase) + } + path := filepath.Join(projectRootDir, casePath) + if !builtin.IsFilePathExists(path) { + return nil, errors.New("referenced testcase file not found: " + path) + } + + refTestCase := TestCasePath(path) + tc, err := refTestCase.ToTestCase() + if err != nil { + return nil, err + } + step.TestCase = tc + testCase.TestSteps = append(testCase.TestSteps, &StepTestCaseWithOptionalArgs{ + step: step, + }) + } else if step.ThinkTime != nil { + testCase.TestSteps = append(testCase.TestSteps, &StepThinkTime{ + step: step, + }) + } else if step.Request != nil { + testCase.TestSteps = append(testCase.TestSteps, &StepRequestWithOptionalArgs{ + step: step, + }) + } else if step.Transaction != nil { + testCase.TestSteps = append(testCase.TestSteps, &StepTransaction{ + step: step, + }) + } else if step.Rendezvous != nil { + testCase.TestSteps = append(testCase.TestSteps, &StepRendezvous{ + step: step, + }) + } else { + log.Warn().Interface("step", step).Msg("[convertTestCase] unexpected step") + } + } + return testCase, nil } diff --git a/hrp/convert_test.go b/hrp/convert_test.go index 6c2d619b..2b0556fb 100644 --- a/hrp/convert_test.go +++ b/hrp/convert_test.go @@ -153,7 +153,7 @@ var demoTestCaseWithoutPlugin = &TestCase{ } func TestGenDemoTestCase(t *testing.T) { - tCase, _ := demoTestCaseWithPlugin.ToTCase() + tCase := demoTestCaseWithPlugin.ToTCase() err := builtin.Dump2JSON(tCase, demoTestCaseWithPluginJSONPath.GetPath()) if err != nil { t.Fail() @@ -163,7 +163,7 @@ func TestGenDemoTestCase(t *testing.T) { t.Fail() } - tCase, _ = demoTestCaseWithoutPlugin.ToTCase() + tCase = demoTestCaseWithoutPlugin.ToTCase() err = builtin.Dump2JSON(tCase, demoTestCaseWithoutPluginJSONPath.GetPath()) if err != nil { t.Fail() diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index d1701bb2..6f8b4442 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -132,6 +132,18 @@ func IsFilePathExists(path string) bool { return true } +// IsFolderPathExists returns true if path exists and path is folder +func IsFolderPathExists(path string) bool { + info, err := os.Stat(path) + if err != nil { + // path not exists + return false + } + + // path exists and is dir + return info.IsDir() +} + func EnsureFolderExists(folderPath string) error { if !IsPathExists(folderPath) { err := CreateFolder(folderPath) diff --git a/hrp/models.go b/hrp/models.go index 4b5068cc..4247cf4a 100644 --- a/hrp/models.go +++ b/hrp/models.go @@ -222,21 +222,19 @@ type IAPI interface { // TStep represents teststep data structure. // Each step maybe two different type: make one HTTP request or reference another testcase. type TStep struct { - Name string `json:"name" yaml:"name"` // required - Request *Request `json:"request,omitempty" yaml:"request,omitempty"` - APIPath string `json:"api,omitempty" yaml:"api,omitempty"` - APIContent IAPI `json:"api_content,omitempty" yaml:"api_content,omitempty"` - TestCasePath string `json:"testcase,omitempty" yaml:"testcase,omitempty"` - TestCaseContent ITestCase `json:"testcase_content,omitempty" yaml:"testcase_content,omitempty"` - Transaction *Transaction `json:"transaction,omitempty" yaml:"transaction,omitempty"` - Rendezvous *Rendezvous `json:"rendezvous,omitempty" yaml:"rendezvous,omitempty"` - ThinkTime *ThinkTime `json:"think_time,omitempty" yaml:"think_time,omitempty"` - Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"` - SetupHooks []string `json:"setup_hooks,omitempty" yaml:"setup_hooks,omitempty"` - TeardownHooks []string `json:"teardown_hooks,omitempty" yaml:"teardown_hooks,omitempty"` - Extract map[string]string `json:"extract,omitempty" yaml:"extract,omitempty"` - Validators []interface{} `json:"validate,omitempty" yaml:"validate,omitempty"` - Export []string `json:"export,omitempty" yaml:"export,omitempty"` + Name string `json:"name" yaml:"name"` // required + Request *Request `json:"request,omitempty" yaml:"request,omitempty"` + API interface{} `json:"api,omitempty" yaml:"api,omitempty"` // *APIPath or *API + TestCase interface{} `json:"testcase,omitempty" yaml:"testcase,omitempty"` // *TestCasePath or *TestCase + Transaction *Transaction `json:"transaction,omitempty" yaml:"transaction,omitempty"` + Rendezvous *Rendezvous `json:"rendezvous,omitempty" yaml:"rendezvous,omitempty"` + ThinkTime *ThinkTime `json:"think_time,omitempty" yaml:"think_time,omitempty"` + Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"` + SetupHooks []string `json:"setup_hooks,omitempty" yaml:"setup_hooks,omitempty"` + TeardownHooks []string `json:"teardown_hooks,omitempty" yaml:"teardown_hooks,omitempty"` + Extract map[string]string `json:"extract,omitempty" yaml:"extract,omitempty"` + Validators []interface{} `json:"validate,omitempty" yaml:"validate,omitempty"` + Export []string `json:"export,omitempty" yaml:"export,omitempty"` } type stepType string @@ -293,10 +291,6 @@ type TCase struct { TestSteps []*TStep `json:"teststeps" yaml:"teststeps"` } -func (tc *TCase) Path() string { - return tc.Config.Path -} - // IStep represents interface for all types for teststeps, includes: // StepRequest, StepRequestWithOptionalArgs, StepRequestValidation, StepRequestExtraction, // StepTestCaseWithOptionalArgs, @@ -312,7 +306,6 @@ type IStep interface { type ITestCase interface { GetPath() string ToTestCase() (*TestCase, error) - ToTCase() (*TCase, error) } // TestCase is a container for one testcase, which is used for testcase runner. @@ -330,14 +323,14 @@ func (tc *TestCase) ToTestCase() (*TestCase, error) { return tc, nil } -func (tc *TestCase) ToTCase() (*TCase, error) { - tCase := TCase{ +func (tc *TestCase) ToTCase() *TCase { + tCase := &TCase{ Config: tc.Config, } for _, step := range tc.TestSteps { tCase.TestSteps = append(tCase.TestSteps, step.ToStruct()) } - return &tCase, nil + return tCase } type testCaseStat struct { diff --git a/hrp/runner.go b/hrp/runner.go index 828cf8d8..b0e50c7c 100644 --- a/hrp/runner.go +++ b/hrp/runner.go @@ -10,6 +10,7 @@ import ( "fmt" "html/template" "io" + "io/ioutil" "net" "net/http" "net/http/httputil" @@ -148,15 +149,17 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error { defer sdk.SendEvent(event.StartTiming("execution")) // record execution data to summary s := newOutSummary() - for _, iTestCase := range testcases { - testcase, err := iTestCase.ToTestCase() - if err != nil { - log.Error().Err(err).Msg("[Run] convert ITestCase interface to TestCase struct failed") - return err - } + + // load all testcases + testCases, err := loadTestCases(testcases...) + if err != nil { + return err + } + + for _, testcase := range testCases { cfg := testcase.Config // parse config parameters - err = initParameterIterator(cfg, "runner") + err := initParameterIterator(cfg, "runner") if err != nil { log.Error().Interface("parameters", cfg.Parameters).Err(err).Msg("parse config parameters failed") return err @@ -201,6 +204,62 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error { return nil } +func loadTestCases(iTestCases ...ITestCase) ([]*TestCase, error) { + testCases := make([]*TestCase, 0) + + for _, iTestCase := range iTestCases { + if _, ok := iTestCase.(*TestCase); ok { + testcase, err := iTestCase.ToTestCase() + if err != nil { + log.Error().Err(err).Msg("failed to convert ITestCase interface to TestCase struct") + return nil, err + } + testCases = append(testCases, testcase) + continue + } + + // iTestCase should be a TestCasePath, file path or folder path + testCasePath, ok := iTestCase.(*TestCasePath) + if !ok { + return nil, errors.New("invalid iTestCase type") + } + + casePaths := make([]*TestCasePath, 0) + casePath := iTestCase.GetPath() + if builtin.IsFolderPathExists(casePath) { + // folder path + files, err := ioutil.ReadDir(casePath) + if err != nil { + return nil, errors.Wrap(err, "read dir failed") + } + for _, f := range files { + ext := filepath.Ext(f.Name()) + if ext != ".yml" && ext != ".yaml" && ext != ".json" { + // ignore non-testcase files + continue + } + path := TestCasePath(filepath.Join(casePath, f.Name())) + casePaths = append(casePaths, &path) + } + } else { + // file path + casePaths = append(casePaths, testCasePath) + } + + for _, path := range casePaths { + tc, err := path.ToTestCase() + if err != nil { + log.Error().Err(err).Str("path", path.GetPath()).Msg("load testcase failed") + return nil, errors.Wrap(err, "load testcase failed") + } + testCases = append(testCases, tc) + } + } + + log.Info().Int("count", len(testCases)).Msg("load testcases successfully") + return testCases, nil +} + func (r *HRPRunner) newCaseRunner(testcase *TestCase) *caseRunner { caseRunner := &caseRunner{ TestCase: testcase, @@ -345,7 +404,7 @@ func (r *caseRunner) runStep(index int, caseConfig *TConfig) (stepResult *stepDa if _, ok := step.(*StepAPIWithOptionalArgs); ok { // run referenced API log.Info().Str("api", copiedStep.Name).Msg("run referenced api") - api, _ := copiedStep.APIContent.ToAPI() + api, _ := copiedStep.API.(*API) extendWithAPI(copiedStep, api) } // override headers @@ -508,7 +567,7 @@ func (r *caseRunner) runStepRendezvous(rendezvous *Rendezvous) (stepResult *step } func (r *caseRunner) isPreRendezvousAllReleased(rendezvous *Rendezvous) bool { - tCase, _ := r.ToTCase() + tCase := r.TestCase.ToTCase() for _, step := range tCase.TestSteps { preRendezvous := step.Rendezvous if preRendezvous == nil { @@ -555,7 +614,7 @@ func (r *Rendezvous) setReleased() { } func initRendezvous(testcase *TestCase, total int64) []*Rendezvous { - tCase, _ := testcase.ToTCase() + tCase := testcase.ToTCase() var rendezvousList []*Rendezvous for _, step := range tCase.TestSteps { if step.Rendezvous == nil { @@ -967,7 +1026,7 @@ func (r *caseRunner) runStepTestCase(step *TStep) (stepResult *stepData, err err StepType: stepTypeTestCase, Success: false, } - testcase := step.TestCaseContent + testcase := step.TestCase.(*TestCase) // copy testcase to avoid data racing copiedTestCase := &TestCase{} diff --git a/hrp/runner_test.go b/hrp/runner_test.go index dcab96c5..85216814 100644 --- a/hrp/runner_test.go +++ b/hrp/runner_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" "github.com/httprunner/httprunner/hrp/internal/scaffold" ) @@ -283,3 +284,37 @@ func TestRunCaseWithRefAPI(t *testing.T) { t.Fail() } } + +func TestLoadTestCases(t *testing.T) { + // load test cases from folder path + tc := TestCasePath("../examples/demo-with-py-plugin/testcases/") + testCases, err := loadTestCases(&tc) + if !assert.Nil(t, err) { + t.Fail() + } + if !assert.Equal(t, len(testCases), 3) { + t.Fail() + } + + // load test cases from single file path + tc = demoTestCaseWithPluginJSONPath + testCases, err = loadTestCases(&tc) + if !assert.Nil(t, err) { + t.Fail() + } + if !assert.Equal(t, len(testCases), 1) { + t.Fail() + } + + // load test cases from TestCase instance + testcase := &TestCase{ + Config: NewConfig("TestCase").SetWeight(3), + } + testCases, err = loadTestCases(testcase) + if !assert.Nil(t, err) { + t.Fail() + } + if !assert.Equal(t, len(testCases), 1) { + t.Fail() + } +} diff --git a/hrp/step.go b/hrp/step.go index f0dd63f9..72da0818 100644 --- a/hrp/step.go +++ b/hrp/step.go @@ -169,7 +169,7 @@ func (s *StepRequest) PATCH(url string) *StepRequestWithOptionalArgs { // CallRefCase calls a referenced testcase. func (s *StepRequest) CallRefCase(tc ITestCase) *StepTestCaseWithOptionalArgs { var err error - s.step.TestCaseContent, err = tc.ToTestCase() + s.step.TestCase, err = tc.ToTestCase() if err != nil { log.Error().Err(err).Msg("failed to load testcase") os.Exit(1) @@ -182,7 +182,7 @@ func (s *StepRequest) CallRefCase(tc ITestCase) *StepTestCaseWithOptionalArgs { // CallRefAPI calls a referenced api. func (s *StepRequest) CallRefAPI(api IAPI) *StepAPIWithOptionalArgs { var err error - s.step.APIContent, err = api.ToAPI() + s.step.API, err = api.ToAPI() if err != nil { log.Error().Err(err).Msg("failed to load api") os.Exit(1) @@ -332,8 +332,10 @@ func (s *StepAPIWithOptionalArgs) TeardownHook(hook string) *StepAPIWithOptional // Export specifies variable names to export from referenced api for current step. func (s *StepAPIWithOptionalArgs) Export(names ...string) *StepAPIWithOptionalArgs { - api, _ := s.step.APIContent.ToAPI() - s.step.Export = append(api.Export, names...) + api, ok := s.step.API.(*API) + if ok { + s.step.Export = append(api.Export, names...) + } return s } @@ -341,8 +343,11 @@ func (s *StepAPIWithOptionalArgs) Name() string { if s.step.Name != "" { return s.step.Name } - api, _ := s.step.APIContent.ToAPI() - return api.Name + api, ok := s.step.API.(*API) + if ok { + return api.Name + } + return "" } func (s *StepAPIWithOptionalArgs) Type() string { @@ -374,8 +379,11 @@ func (s *StepTestCaseWithOptionalArgs) Name() string { if s.step.Name != "" { return s.step.Name } - ts, _ := s.step.TestCaseContent.ToTestCase() - return ts.Config.Name + ts, ok := s.step.TestCase.(*TestCase) + if ok { + return ts.Config.Name + } + return "" } func (s *StepTestCaseWithOptionalArgs) Type() string {