diff --git a/boomer.go b/boomer.go index 62b8933e..0976d134 100644 --- a/boomer.go +++ b/boomer.go @@ -3,6 +3,7 @@ package hrp import ( "time" + "github.com/jinzhu/copier" "github.com/rs/zerolog/log" "github.com/httprunner/hrp/internal/boomer" @@ -46,33 +47,43 @@ func (b *hrpBoomer) Run(testcases ...ITestCase) { panic(err) } cfg := testcase.Config.ToStruct() - parameters := getParameters(testcase.Config) - if parameters == nil { - parameters = []map[string]interface{}{{}} - } - for _, parameter := range parameters { - cfg.Variables = mergeVariables(parameter, cfg.Variables) - task := b.convertBoomerTask(testcase) - taskSlice = append(taskSlice, task) + err = initParameterIterator(cfg, "boomer") + if err != nil { + panic(err) } + task := b.convertBoomerTask(testcase) + taskSlice = append(taskSlice, task) } b.Boomer.Run(taskSlice...) } func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task { + hrpRunner := NewRunner(nil).SetDebug(b.debug) config := testcase.Config.ToStruct() return &boomer.Task{ Name: config.Name, Weight: config.Weight, Fn: func() { - runner := NewRunner(nil).SetDebug(b.debug).Reset() + runner := hrpRunner.newCaseRunner(testcase) testcaseSuccess := true // flag whole testcase result var transactionSuccess = true // flag current transaction result + cfg := testcase.Config.ToStruct() + caseConfig := &TConfig{} + // copy config to avoid data racing + if err := copier.Copy(caseConfig, cfg); err != nil { + log.Error().Err(err).Msg("copy config data failed") + } + // iterate through all parameter iterators and update case variables + for _, it := range caseConfig.ParametersSetting.Iterators { + if it.HasNext() { + caseConfig.Variables = mergeVariables(it.Next(), caseConfig.Variables) + } + } startTime := time.Now() - for _, step := range testcase.TestSteps { - stepData, err := runner.runStep(step, testcase.Config) + for index, step := range testcase.TestSteps { + stepData, err := runner.runStep(index, caseConfig) if err != nil { // step failed var elapsed int64 @@ -85,8 +96,8 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task { testcaseSuccess = false transactionSuccess = false - if runner.failfast { - log.Error().Err(err).Msg("abort running due to failfast setting") + if runner.hrpRunner.failfast { + log.Error().Msg("abort running due to failfast setting") break } log.Warn().Err(err).Msg("run step failed, continue next step") @@ -115,7 +126,7 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task { for name, transaction := range runner.transactions { if len(transaction) == 1 { // if transaction end time not exists, use testcase end time instead - duration := endTime.Sub(transaction[TransactionStart]) + duration := endTime.Sub(transaction[transactionStart]) b.RecordTransaction(name, transactionSuccess, duration.Milliseconds(), 0) } } diff --git a/convert.go b/convert.go index 7cdedfae..7f1bc830 100644 --- a/convert.go +++ b/convert.go @@ -4,10 +4,11 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/rs/zerolog/log" - "gopkg.in/yaml.v3" "io/ioutil" "path/filepath" + + "github.com/rs/zerolog/log" + "gopkg.in/yaml.v3" ) func (tc *TCase) Dump2JSON(path string) error { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 67354670..4f2f8207 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,15 +1,19 @@ # Release History -## v0.3.0 (2021-12-22) +## v0.3.1 (2021-12-30) + +- feat: set ulimit to 10240 before load testing +- fix: concurrent map writes in load testing + +## v0.3.0 (2021-12-24) - feat: implement `transaction` mechanism for load test - feat: continue running next step when failure occurs with `--continue-on-failure` flag, default to failfast -- feat: spawn workers with `--spawn-rate` flag -- refactor: fork [boomer] as sub module - feat: report GA events with version -- feat: run load test with the given limit and burst as rate limiter +- feat: run load test with the given limit and burst as rate limiter, use `--spawn-count`, `--spawn-rate` and `--request-increase-rate` flag +- feat: report runner state to prometheus +- refactor: fork [boomer] as submodule initially and made a lot of changes - change: update API models -- feat: report runner state ## v0.2.2 (2021-12-07) diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index b2067d4a..8b751a2d 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -4,9 +4,19 @@ One-stop solution for HTTP(S) testing. ### Synopsis -hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, load testing and digital experience monitoring (DEM). Enjoy! โœจ ๐Ÿš€ โœจ + +โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— +โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ•šโ•โ•โ–ˆโ–ˆโ•”โ•โ•โ•โ•šโ•โ•โ–ˆโ–ˆโ•”โ•โ•โ•โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•— +โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ• +โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•”โ•โ•โ•โ• โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•— +โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ +โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ• โ•šโ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•โ•โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ• + +hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, +load testing and digital experience monitoring (DEM). Enjoy! โœจ ๐Ÿš€ โœจ License: Apache-2.0 +Website: https://httprunner.com Github: https://github.com/httprunner/hrp Copyright 2021 debugtalk @@ -22,4 +32,4 @@ Copyright 2021 debugtalk * [hrp har2case](hrp_har2case.md) - Convert HAR to json/yaml testcase files * [hrp run](hrp_run.md) - run API test -###### Auto generated by spf13/cobra on 24-Dec-2021 +###### Auto generated by spf13/cobra on 30-Dec-2021 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index c5a65c42..494ae8d6 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -38,4 +38,4 @@ hrp boom [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 24-Dec-2021 +###### Auto generated by spf13/cobra on 30-Dec-2021 diff --git a/docs/cmd/hrp_har2case.md b/docs/cmd/hrp_har2case.md index 8e559e4b..d9664517 100644 --- a/docs/cmd/hrp_har2case.md +++ b/docs/cmd/hrp_har2case.md @@ -23,4 +23,4 @@ hrp har2case harPath... [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 24-Dec-2021 +###### Auto generated by spf13/cobra on 30-Dec-2021 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index 96cd3590..9cb28dbf 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -31,4 +31,4 @@ hrp run path... [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 24-Dec-2021 +###### Auto generated by spf13/cobra on 30-Dec-2021 diff --git a/examples/parameters_test.json b/examples/parameters_test.json index 6ad52032..1a555593 100644 --- a/examples/parameters_test.json +++ b/examples/parameters_test.json @@ -6,14 +6,18 @@ "iOS/10.1", "iOS/10.2" ], - "username-password": "${parameterize(examples/account.csv)}", - "app_version": "${getAppVersion()}" + "username-password": "${parameterize(examples/account.csv)}" }, "parameters_setting": { - "strategy": "random" + "strategy": { + "user_agent": "sequential", + "username-password": "random" + }, + "iteration": 6 }, "variables": { - "app_version": "f1" + "app_version": "v1", + "user_agent": "iOS/10.3" }, "base_url": "https://postman-echo.com", "verify": false @@ -24,7 +28,7 @@ "variables": { "foo1": "$username", "foo2": "$password", - "foo3": "$app_version" + "foo3": "$user_agent" }, "request": { "method": "GET", @@ -48,7 +52,7 @@ { "check": "body.args.foo3", "assert": "not_equal", - "expect": "f1", + "expect": "iOS/10.3", "msg": "check app version" } ] diff --git a/examples/parameters_test.yaml b/examples/parameters_test.yaml index 7afb4252..e2cabc91 100644 --- a/examples/parameters_test.yaml +++ b/examples/parameters_test.yaml @@ -1,38 +1,40 @@ config: - name: "request methods testcase: validate with parameters" - parameters: - user_agent: ["iOS/10.1", "iOS/10.2"] - username-password: ${parameterize(examples/account.csv)} - app_version: ${getAppVersion()} - parameters_setting: - strategy: random - variables: - app_version: f1 - base_url: "https://postman-echo.com" - verify: False + name: "request methods testcase: validate with parameters" + parameters: + user_agent: [ "iOS/10.1", "iOS/10.2" ] + username-password: ${parameterize(examples/account.csv)} + parameters_setting: + strategy: + user_agent: "sequential" + username-password: "random" + iteration: 6 + variables: + app_version: v1 + user_agent: iOS/10.3 + base_url: "https://postman-echo.com" + verify: False teststeps: -- - name: get with params + - name: get with params variables: - foo1: $username - foo2: $password - foo3: $app_version + foo1: $username + foo2: $password + foo3: $user_agent request: - method: GET - url: /get - params: - foo1: $foo1 - foo2: $foo2 - foo3: $foo3 - headers: - User-Agent: $user_agent,$app_version + method: GET + url: /get + params: + foo1: $foo1 + foo2: $foo2 + foo3: $foo3 + headers: + User-Agent: $user_agent,$app_version validate: - - check: status_code - assert: equals - expect: 200 - msg: check status code - - check: body.args.foo3 - assert: not_equal - expect: f1 - msg: check app version \ No newline at end of file + - check: status_code + assert: equals + expect: 200 + msg: check status code + - check: body.args.foo3 + assert: not_equal + expect: iOS/10.3 + msg: check app version \ No newline at end of file diff --git a/hrp/cmd/boom.go b/hrp/cmd/boom.go index 41c67758..b19c239a 100644 --- a/hrp/cmd/boom.go +++ b/hrp/cmd/boom.go @@ -19,7 +19,8 @@ var boomCmd = &cobra.Command{ $ hrp boom examples/ # run testcases in specified folder`, Args: cobra.MinimumNArgs(1), PreRun: func(cmd *cobra.Command, args []string) { - setLogLevel("WARN") // disable info logs for load testing + boomer.SetUlimit(10240) // ulimit -n 10240 + setLogLevel("WARN") // disable info logs for load testing }, Run: func(cmd *cobra.Command, args []string) { var paths []hrp.ITestCase diff --git a/hrp/cmd/har2case.go b/hrp/cmd/har2case.go index 73670a57..7512f241 100644 --- a/hrp/cmd/har2case.go +++ b/hrp/cmd/har2case.go @@ -4,7 +4,7 @@ import ( "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "github.com/httprunner/hrp/har2case" + "github.com/httprunner/hrp/internal/har2case" ) // har2caseCmd represents the har2case command diff --git a/hrp/cmd/root.go b/hrp/cmd/root.go index 2f4d866d..db06388e 100644 --- a/hrp/cmd/root.go +++ b/hrp/cmd/root.go @@ -15,9 +15,19 @@ import ( var RootCmd = &cobra.Command{ Use: "hrp", Short: "One-stop solution for HTTP(S) testing.", - Long: `hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, load testing and digital experience monitoring (DEM). Enjoy! โœจ ๐Ÿš€ โœจ + Long: ` +โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— +โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ•šโ•โ•โ–ˆโ–ˆโ•”โ•โ•โ•โ•šโ•โ•โ–ˆโ–ˆโ•”โ•โ•โ•โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ•โ•โ•โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•— +โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ–ˆโ–ˆโ•— โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•— โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ• +โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•”โ•โ•โ•โ• โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•”โ•โ•โ• โ–ˆโ–ˆโ•”โ•โ•โ–ˆโ–ˆโ•— +โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•”โ•โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ•‘ โ•šโ–ˆโ–ˆโ–ˆโ–ˆโ•‘โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ•—โ–ˆโ–ˆโ•‘ โ–ˆโ–ˆโ•‘ +โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ• โ•šโ•โ•โ•โ•โ•โ• โ•šโ•โ• โ•šโ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ•โ•โ•โ•šโ•โ•โ•โ•โ•โ•โ•โ•šโ•โ• โ•šโ•โ• + +hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, +load testing and digital experience monitoring (DEM). Enjoy! โœจ ๐Ÿš€ โœจ License: Apache-2.0 +Website: https://httprunner.com Github: https://github.com/httprunner/hrp Copyright 2021 debugtalk`, PersistentPreRun: func(cmd *cobra.Command, args []string) { diff --git a/internal/boomer/ulimit.go b/internal/boomer/ulimit.go new file mode 100644 index 00000000..b83585ea --- /dev/null +++ b/internal/boomer/ulimit.go @@ -0,0 +1,32 @@ +// +build !windows + +package boomer + +import ( + "syscall" + + "github.com/rs/zerolog/log" +) + +// set resource limit +// ulimit -n 10240 +func SetUlimit(limit uint64) { + var rLimit syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) + if err != nil { + log.Error().Err(err).Msg("get ulimit failed") + return + } + log.Info().Uint64("limit", rLimit.Cur).Msg("get current ulimit") + if rLimit.Cur >= limit { + return + } + + rLimit.Cur = limit + log.Info().Uint64("limit", rLimit.Cur).Msg("set current ulimit") + err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit) + if err != nil { + log.Error().Err(err).Msg("set ulimit failed") + return + } +} diff --git a/internal/boomer/ulimit_windows.go b/internal/boomer/ulimit_windows.go new file mode 100644 index 00000000..76ca69fc --- /dev/null +++ b/internal/boomer/ulimit_windows.go @@ -0,0 +1,12 @@ +// +build windows + +package boomer + +import ( + "github.com/rs/zerolog/log" +) + +// set resource limit +func SetUlimit(limit uint64) { + log.Warn().Msg("windows does not support setting ulimit") +} diff --git a/internal/builtin/function.go b/internal/builtin/function.go index 3d5c96de..f4f7aaae 100644 --- a/internal/builtin/function.go +++ b/internal/builtin/function.go @@ -4,13 +4,14 @@ import ( "crypto/md5" "encoding/csv" "encoding/hex" - "github.com/rs/zerolog/log" "io/ioutil" "math" "math/rand" "path/filepath" "strings" "time" + + "github.com/rs/zerolog/log" ) var Functions = map[string]interface{}{ @@ -19,8 +20,8 @@ var Functions = map[string]interface{}{ "gen_random_string": genRandomString, // call with one argument "max": math.Max, // call with two arguments "md5": MD5, - "parameterize": LoadFromCSV, - "P": LoadFromCSV, + "parameterize": loadFromCSV, + "P": loadFromCSV, } func init() { @@ -52,7 +53,7 @@ func MD5(str string) string { return hex.EncodeToString(hasher.Sum(nil)) } -func LoadFromCSV(path string) []map[string]string { +func loadFromCSV(path string) []map[string]interface{} { path, err := filepath.Abs(path) if err != nil { log.Error().Str("path", path).Err(err).Msg("convert absolute path failed") @@ -71,9 +72,9 @@ func LoadFromCSV(path string) []map[string]string { log.Error().Err(err).Msg("parse csv file failed") panic(err) } - var result []map[string]string + var result []map[string]interface{} for i := 1; i < len(content); i++ { - row := make(map[string]string) + row := make(map[string]interface{}) for j := 0; j < len(content[i]); j++ { row[content[0][j]] = content[i][j] } diff --git a/har2case/README.md b/internal/har2case/README.md similarity index 100% rename from har2case/README.md rename to internal/har2case/README.md diff --git a/har2case/core.go b/internal/har2case/core.go similarity index 100% rename from har2case/core.go rename to internal/har2case/core.go diff --git a/har2case/core_test.go b/internal/har2case/core_test.go similarity index 96% rename from har2case/core_test.go rename to internal/har2case/core_test.go index 43118e58..0e17cfe2 100644 --- a/har2case/core_test.go +++ b/internal/har2case/core_test.go @@ -7,8 +7,8 @@ import ( ) var ( - harPath = "../examples/har/demo.har" - harPath2 = "../examples/har/postman-echo.har" + harPath = "../../examples/har/demo.har" + harPath2 = "../../examples/har/postman-echo.har" ) func TestGenJSON(t *testing.T) { diff --git a/har2case/har.go b/internal/har2case/har.go similarity index 100% rename from har2case/har.go rename to internal/har2case/har.go diff --git a/internal/version/init.go b/internal/version/init.go index a29e664d..462025a5 100644 --- a/internal/version/init.go +++ b/internal/version/init.go @@ -1,3 +1,3 @@ package version -const VERSION = "v0.3.0" +const VERSION = "v0.4.0" diff --git a/models.go b/models.go index 703623fc..b7c7b915 100644 --- a/models.go +++ b/models.go @@ -1,5 +1,11 @@ package hrp +import ( + "math/rand" + "sync" + "time" +) + const ( httpGET string = "GET" httpHEAD string = "HEAD" @@ -18,11 +24,65 @@ type TConfig struct { BaseURL string `json:"base_url,omitempty" yaml:"base_url,omitempty"` Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"` Parameters map[string]interface{} `json:"parameters,omitempty" yaml:"parameters,omitempty"` - ParametersSetting map[string]interface{} `json:"parameters_setting,omitempty" yaml:"parameters_setting,omitempty"` + ParametersSetting *TParamsConfig `json:"parameters_setting,omitempty" yaml:"parameters_setting,omitempty"` Export []string `json:"export,omitempty" yaml:"export,omitempty"` Weight int `json:"weight,omitempty" yaml:"weight,omitempty"` } +type TParamsConfig struct { + Strategy interface{} `json:"strategy,omitempty" yaml:"strategy,omitempty"` + Iteration int `json:"iteration,omitempty" yaml:"iteration,omitempty"` + Iterators []*Iterator `json:"parameterIterator,omitempty" yaml:"parameterIterator,omitempty"` //ไฟๅญ˜ๅ‚ๆ•ฐ็š„่ฟญไปฃๅ™จ +} + +const ( + strategyRandom string = "random" + strategySequential string = "Sequential" +) + +type paramsType []map[string]interface{} + +type Iterator struct { + sync.Mutex + data paramsType + strategy string // random, sequential + iteration int + index int +} + +func (params paramsType) Iterator() *Iterator { + return &Iterator{ + data: params, + iteration: len(params), + index: 0, + } +} + +func (iter *Iterator) HasNext() bool { + if iter.iteration == -1 { + return true + } + return iter.index < iter.iteration +} + +func (iter *Iterator) Next() (value map[string]interface{}) { + iter.Lock() + defer iter.Unlock() + if len(iter.data) == 0 { + iter.index++ + return map[string]interface{}{} + } + if iter.strategy == strategyRandom { + randSource := rand.New(rand.NewSource(time.Now().Unix())) + randIndex := randSource.Intn(len(iter.data)) + value = iter.data[randIndex] + } else { + value = iter.data[iter.index%len(iter.data)] + } + iter.index++ + return value +} + // Request represents HTTP request data structure. // This is used for teststep. type Request struct { @@ -70,16 +130,16 @@ const ( stepTypeRendezvous stepType = "rendezvous" ) -type TransactionType string +type transactionType string const ( - TransactionStart TransactionType = "start" - TransactionEnd TransactionType = "end" + transactionStart transactionType = "start" + transactionEnd transactionType = "end" ) type Transaction struct { Name string `json:"name" yaml:"name"` - Type TransactionType `json:"type" yaml:"type"` + Type transactionType `json:"type" yaml:"type"` } type Rendezvous struct { Name string `json:"name" yaml:"name"` // required diff --git a/parser.go b/parser.go index 7bab526d..ef10fab9 100644 --- a/parser.go +++ b/parser.go @@ -3,12 +3,10 @@ package hrp import ( "encoding/json" "fmt" - "math/rand" "net/url" "reflect" "regexp" "strings" - "time" "github.com/maja42/goval" "github.com/pkg/errors" @@ -496,27 +494,18 @@ func findallVariables(raw string) variableSet { return varSet } -func shuffleCartesianProduct(slice []map[string]interface{}) { - if slice == nil || len(slice) == 0 { - return - } - r := rand.New(rand.NewSource(time.Now().Unix())) - for len(slice) > 0 { - n := len(slice) - randIndex := r.Intn(n) - slice[n-1], slice[randIndex] = slice[randIndex], slice[n-1] - slice = slice[:n-1] - } -} - -func genCartesianProduct(params [][]map[string]interface{}) []map[string]interface{} { - if params == nil || len(params) == 0 { +func genCartesianProduct(paramsMap map[string]paramsType) paramsType { + if len(paramsMap) == 0 { return nil } - var cartesianProduct []map[string]interface{} + var params []paramsType + for _, v := range paramsMap { + params = append(params, v) + } + var cartesianProduct paramsType cartesianProduct = params[0] for i := 0; i < len(params)-1; i++ { - var tempProduct []map[string]interface{} + var tempProduct paramsType for _, param1 := range cartesianProduct { for _, param2 := range params[i+1] { tempProduct = append(tempProduct, mergeVariables(param1, param2)) @@ -527,103 +516,156 @@ func genCartesianProduct(params [][]map[string]interface{}) []map[string]interfa return cartesianProduct } -func getParameters(config IConfig) []map[string]interface{} { - cfg := config.ToStruct() - // parse config parameters - parsedParams, err := parseParameters(cfg.Parameters, cfg.Variables) - if err != nil { - log.Error().Interface("parameters", cfg.Parameters).Err(err).Msg("parse config parameters failed") - } - if cfg.ParametersSetting["strategy"] != nil && strings.ToLower(cfg.ParametersSetting["strategy"].(string)) == "random" { - shuffleCartesianProduct(parsedParams) - } - return parsedParams -} - -func parseParameters(parameters map[string]interface{}, variablesMapping map[string]interface{}) ([]map[string]interface{}, error) { - if parameters == nil || len(parameters) == 0 { +func parseParameters(parameters map[string]interface{}, variablesMapping map[string]interface{}) (map[string]paramsType, error) { + if len(parameters) == 0 { return nil, nil } - var parsedParametersSlice [][]map[string]interface{} + parsedParametersSlice := make(map[string]paramsType) + var err error for k, v := range parameters { - parameterNameSlice := strings.Split(k, "-") - var parameterSlice []map[string]interface{} + var parameterSlice paramsType rawValue := reflect.ValueOf(v) switch rawValue.Kind() { case reflect.String: - parsedParameterContent, err := parseData(rawValue.Interface(), variablesMapping) + // e.g. username-password: ${parameterize(examples/account.csv)} -> [{"username": "test1", "password": "111111"}, {"username": "test2", "password": "222222"}] + var parsedParameterContent interface{} + parsedParameterContent, err = parseString(rawValue.String(), variablesMapping) if err != nil { - log.Error().Interface("parameter", parameters).Msg("[parseParameters] parse parameter error") + log.Error().Interface("parameterContent", rawValue).Msg("[parseParameters] parse parameter content error") return nil, err } parsedParameterRawValue := reflect.ValueOf(parsedParameterContent) if parsedParameterRawValue.Kind() != reflect.Slice { - log.Error().Interface("parameter", parameters).Msg("[parseParameters] parsed parameter content should be Slice, got %v") - return nil, errors.New("parsed parameter content should be Slice") + log.Error().Interface("parameterContent", parsedParameterRawValue).Msg("[parseParameters] parsed parameter content should be slice") + return nil, errors.New("parsed parameter content should be slice") } - for i := 0; i < parsedParameterRawValue.Len(); i++ { - parameterMap := make(map[string]interface{}) - // e.g. - elem := reflect.ValueOf(parsedParameterRawValue.Index(i).Interface()) - if elem.Kind() == reflect.Map { - // e.g. [{"username": "test1", "password": "passwd1", "other": "111"}, {"username": "test2", "password": "passwd2", "other": ""222}] - // -> [{"username": "test1", "password": "passwd1"}, {"username": "test2", "password": "passwd2"}] (username, password in parameterNameSlice) - for _, key := range parameterNameSlice { - if _, ok := elem.Interface().(map[string]string)[key]; ok { - parameterMap[key] = elem.MapIndex(reflect.ValueOf(key)).Interface() - } else { - log.Error().Interface("parameterNameSlice", parameterNameSlice).Msg("[parseParameters] parameter name not found") - return nil, errors.New("parameter name not found") - } - } - } else if elem.Kind() == reflect.Slice { - // e.g. [["test1", "passwd1"], ["test2", "passwd2"]] -> [{"username": "test1", "password": "passwd1"}, {"username": "test2", "password": "passwd2"}] - if len(parameterNameSlice) != elem.Len() { - log.Error().Interface("parameter", parameters).Msg("[parseParameters] parameter name Slice and parameter content Slice should have the same length") - return nil, errors.New("parameter name Slice and parameter cjntent Slice should have the same length") - } else { - for j := 0; j < elem.Len(); j++ { - parameterMap[parameterNameSlice[j]] = elem.Index(j).Interface() - } - } + parameterSlice, err = parseSlice(k, parsedParameterRawValue.Interface()) + case reflect.Slice: + // e.g. user_agent: ["iOS/10.1", "iOS/10.2"] -> [{"user_agent": "iOS/10.1"}, {"user_agent": "iOS/10.2"}] + parameterSlice, err = parseSlice(k, rawValue.Interface()) + default: + log.Error().Interface("parameter", parameters).Msg("[parseParameters] parameter content should be slice or text(functions call)") + return nil, errors.New("parameter content should be slice or text(functions call)") + } + if err != nil { + return nil, err + } + parsedParametersSlice[k] = parameterSlice + } + return parsedParametersSlice, nil +} + +func parseSlice(parameterName string, parameterContent interface{}) ([]map[string]interface{}, error) { + parameterNameSlice := strings.Split(parameterName, "-") + var parameterSlice []map[string]interface{} + parameterContentSlice := reflect.ValueOf(parameterContent) + if parameterContentSlice.Kind() != reflect.Slice { + return nil, errors.New("parameterContent should be slice") + } + for i := 0; i < parameterContentSlice.Len(); i++ { + parameterMap := make(map[string]interface{}) + elem := reflect.ValueOf(parameterContentSlice.Index(i).Interface()) + switch elem.Kind() { + case reflect.Map: + // e.g. "username-password": [{"username": "test1", "password": "passwd1", "other": "111"}, {"username": "test2", "password": "passwd2", "other": ""222}] + // -> [{"username": "test1", "password": "passwd1"}, {"username": "test2", "password": "passwd2"}] + for _, key := range parameterNameSlice { + if _, ok := elem.Interface().(map[string]interface{})[key]; ok { + parameterMap[key] = elem.MapIndex(reflect.ValueOf(key)).Interface() } else { - // e.g. ${getAppVersion()} -> [3.1, 3.0] -> [{"app_version": 3.1}, {"app_version": 3.0}] - if len(parameterNameSlice) != 1 { - log.Error().Interface("parameterNameSlice", parameterNameSlice).Msg("[parseParameters] parameter name slice should have only one element when parameter content is string") - return nil, errors.New("parameter name slice should have only one element when parameter content is string") - } - parameterMap[parameterNameSlice[0]] = elem.Interface() + log.Error().Interface("parameterNameSlice", parameterNameSlice).Msg("[parseParameters] parameter name not found") + return nil, errors.New("parameter name not found") } - parameterSlice = append(parameterSlice, parameterMap) } case reflect.Slice: - for i := 0; i < rawValue.Len(); i++ { - parameterMap := make(map[string]interface{}) - elem := reflect.ValueOf(rawValue.Index(i).Interface()) - if elem.Kind() == reflect.Slice { - // e.g. username-password: [["test1", "passwd1"], ["test2", "passwd2"]] - if len(parameterNameSlice) != elem.Len() { - log.Error().Interface("parameter", parameters).Msg("[parseParameters] parameter name Slice and parameter content Slice should have the same length") - return nil, errors.New("parameter name Slice and parameter content Slice should have the same length") - } - for j := 0; j < elem.Len(); j++ { - parameterMap[parameterNameSlice[j]] = elem.Index(j).Interface() - } - } else { - // e.g. user_agent: ["iOS/10.1", "iOS/10.2"] - if len(parameterNameSlice) != 1 { - log.Error().Interface("parameterNameSlice", parameterNameSlice).Msg("[parseParameters] parameter name slice should have only one element when parameter content is string") - return nil, errors.New("parameter name slice should have only one element when parameter content is string") - } - parameterMap[parameterNameSlice[0]] = elem.Interface() + // e.g. "username-password": [["test1", "passwd1"], ["test2", "passwd2"]] + // -> [{"username": "test1", "password": "passwd1"}, {"username": "test2", "password": "passwd2"}] + if len(parameterNameSlice) != elem.Len() { + log.Error().Interface("parameterNameSlice", parameterNameSlice).Interface("parameterContent", elem.Interface()).Msg("[parseParameters] parameter name slice and parameter content slice should have the same length") + return nil, errors.New("parameter name slice and parameter content slice should have the same length") + } else { + for j := 0; j < elem.Len(); j++ { + parameterMap[parameterNameSlice[j]] = elem.Index(j).Interface() } - parameterSlice = append(parameterSlice, parameterMap) } default: - log.Error().Interface("parameter", parameters).Msg("[parseParameters] parameter content should be Slice or Text(variables or functions call)") - return nil, errors.New("parameter content should be Slice or Text(variables or functions call)") + // e.g. "app_version": [3.1, 3.0] + // -> [{"app_version": 3.1}, {"app_version": 3.0}] + if len(parameterNameSlice) != 1 { + log.Error().Interface("parameterNameSlice", parameterNameSlice).Msg("[parseParameters] parameter name slice should have only one element when parameter content is string") + return nil, errors.New("parameter name slice should have only one element when parameter content is string") + } + parameterMap[parameterNameSlice[0]] = elem.Interface() } - parsedParametersSlice = append(parsedParametersSlice, parameterSlice) + parameterSlice = append(parameterSlice, parameterMap) } - return genCartesianProduct(parsedParametersSlice), nil + return parameterSlice, nil +} + +func initParameterIterator(cfg *TConfig, mode string) (err error) { + var parameters map[string]paramsType + parameters, err = parseParameters(cfg.Parameters, cfg.Variables) + if err != nil { + return err + } + // parse config parameters setting + if cfg.ParametersSetting == nil { + cfg.ParametersSetting = &TParamsConfig{Iterators: []*Iterator{}} + } + // boomerๆจกๅผไธ‹ไธ้™ๅˆถ่ฟญไปฃๆฌกๆ•ฐ + if mode == "boomer" { + cfg.ParametersSetting.Iteration = -1 + } + rawValue := reflect.ValueOf(cfg.ParametersSetting.Strategy) + switch rawValue.Kind() { + case reflect.Map: + // strategy: {"user_agent": "sequential", "username-password": "random"}, ๆฏไธชๅ‚ๆ•ฐๅฏนๅบ”ไธ€ไธช่ฟญไปฃๅ™จ๏ผŒๆฏไธช่ฟญไปฃๅ™จ้šๆœบใ€้กบๅบ้€‰ๅ–ๅ…ƒ็ด ไบ’ไธๅฝฑๅ“ + for k, v := range parameters { + if _, ok := rawValue.Interface().(map[string]interface{})[k]; ok { + // use strategy if configured + cfg.ParametersSetting.Iterators = append( + cfg.ParametersSetting.Iterators, + newIterator(v, rawValue.MapIndex(reflect.ValueOf(k)).Interface().(string), cfg.ParametersSetting.Iteration), + ) + } else { + // use sequential strategy by default + cfg.ParametersSetting.Iterators = append( + cfg.ParametersSetting.Iterators, + newIterator(v, strategySequential, cfg.ParametersSetting.Iteration), + ) + } + } + case reflect.String: + // strategy: random, ไป…็”Ÿๆˆไธ€ไธช็š„่ฟญไปฃๅ™จ๏ผŒ่ฏฅ่ฟญไปฃๅ™จๅœจๅ‚ๆ•ฐ็ฌ›ๅกๅฐ”็งฏsliceไธญ้šๆœบ้€‰ๅ–ๅ…ƒ็ด  + if len(rawValue.String()) == 0 { + cfg.ParametersSetting.Strategy = strategySequential + } else { + cfg.ParametersSetting.Strategy = strings.ToLower(rawValue.String()) + } + cfg.ParametersSetting.Iterators = append( + cfg.ParametersSetting.Iterators, + newIterator(genCartesianProduct(parameters), cfg.ParametersSetting.Strategy.(string), cfg.ParametersSetting.Iteration), + ) + default: + // default strategy: sequential, ไป…็”Ÿๆˆไธ€ไธช็š„่ฟญไปฃๅ™จ๏ผŒ่ฏฅ่ฟญไปฃๅ™จๅœจๅ‚ๆ•ฐ็ฌ›ๅกๅฐ”็งฏsliceไธญ้กบๅบ้€‰ๅ–ๅ…ƒ็ด  + cfg.ParametersSetting.Strategy = strategySequential + cfg.ParametersSetting.Iterators = append( + cfg.ParametersSetting.Iterators, + newIterator(genCartesianProduct(parameters), cfg.ParametersSetting.Strategy.(string), cfg.ParametersSetting.Iteration), + ) + } + return nil +} + +func newIterator(parameters paramsType, strategy string, iteration int) *Iterator { + iter := parameters.Iterator() + iter.strategy = strategy + if iteration > 0 { + iter.iteration = iteration + } else if iteration < 0 { + iter.iteration = -1 + } else if iter.iteration == 0 { + iter.iteration = 1 + } + return iter } diff --git a/parser_test.go b/parser_test.go index cb1bcd8f..f64ef7fc 100644 --- a/parser_test.go +++ b/parser_test.go @@ -621,44 +621,40 @@ func TestFindallVariables(t *testing.T) { func TestParseParameters(t *testing.T) { testData := []struct { - rawVars map[string]interface{} - expectVars []map[string]interface{} + rawVars map[string]interface{} + expectLength int }{ { map[string]interface{}{ "username-password": "${parameterize(examples/account.csv)}", "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}}, - []map[string]interface{}{ - {"username": "test1", "password": "111111", "user_agent": "IOS/10.1"}, - {"username": "test1", "password": "111111", "user_agent": "IOS/10.2"}, - {"username": "test2", "password": "222222", "user_agent": "IOS/10.1"}, - {"username": "test2", "password": "222222", "user_agent": "IOS/10.2"}, - {"username": "test3", "password": "333333", "user_agent": "IOS/10.1"}, - {"username": "test3", "password": "333333", "user_agent": "IOS/10.2"}}, + 6, }, { map[string]interface{}{ "username-password": [][]interface{}{{"test1", "111111"}, {"test2", "222222"}, {"test3", "333333"}}, "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}, "app_version": []interface{}{0.3}}, - []map[string]interface{}{ - {"username": "test1", "password": "111111", "user_agent": "IOS/10.1", "app_version": 0.3}, - {"username": "test1", "password": "111111", "user_agent": "IOS/10.2", "app_version": 0.3}, - {"username": "test2", "password": "222222", "user_agent": "IOS/10.1", "app_version": 0.3}, - {"username": "test2", "password": "222222", "user_agent": "IOS/10.2", "app_version": 0.3}, - {"username": "test3", "password": "333333", "user_agent": "IOS/10.1", "app_version": 0.3}, - {"username": "test3", "password": "333333", "user_agent": "IOS/10.2", "app_version": 0.3}}, + 6, }, { - map[string]interface{}{}, nil, + map[string]interface{}{ + "username-password": [][]interface{}{{"test1", "111111"}, {"test2", "222222"}, {"test3", "333333"}}, + "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}, + "app_version": []interface{}{0.3, 0.4, 0.5}}, + 18, }, { - nil, nil, + map[string]interface{}{}, 0, + }, + { + nil, 0, }, } for _, data := range testData { - value, _ := parseParameters(data.rawVars, map[string]interface{}{}) - if !assert.Equal(t, data.expectVars, value) { + params, _ := parseParameters(data.rawVars, map[string]interface{}{}) + value := genCartesianProduct(params) + if !assert.Len(t, value, data.expectLength) { t.Fail() } } @@ -691,3 +687,64 @@ func TestParseParametersError(t *testing.T) { } } } + +func TestParseSlice(t *testing.T) { + testData := []struct { + rawVar1 string + rawVar2 interface{} + expect []map[string]interface{} + }{ + { + "username-password", + []map[string]interface{}{{"username": "test1", "password": 111111, "other": "111"}, {"username": "test2", "password": 222222, "other": "222"}}, + []map[string]interface{}{ + {"username": "test1", "password": 111111}, + {"username": "test2", "password": 222222}, + }, + }, + { + "username-password", + [][]string{{"test1", "111111"}, {"test2", "222222"}}, + []map[string]interface{}{ + {"username": "test1", "password": "111111"}, + {"username": "test2", "password": "222222"}, + }, + }, + { + "app_version", + []float64{3.1, 3.0}, + []map[string]interface{}{ + {"app_version": 3.1}, + {"app_version": 3.0}, + }, + }, + } + for _, data := range testData { + value, _ := parseSlice(data.rawVar1, data.rawVar2) + if !assert.Equal(t, data.expect, value) { + t.Fail() + } + } +} + +func TestParseSliceError(t *testing.T) { + testData := []struct { + rawVar1 string + rawVar2 interface{} + }{ + { + "app_version", + 123, + }, + { + "app_version", + "123", + }, + } + for _, data := range testData { + _, err := parseSlice(data.rawVar1, data.rawVar2) + if !assert.Error(t, err) { + t.Fail() + } + } +} diff --git a/runner.go b/runner.go index fea5b69b..8faf84ef 100644 --- a/runner.go +++ b/runner.go @@ -42,30 +42,14 @@ func NewRunner(t *testing.T) *hrpRunner { }, Timeout: 30 * time.Second, }, - sessionVariables: make(map[string]interface{}), - transactions: make(map[string]map[TransactionType]time.Time), } } type hrpRunner struct { - t *testing.T - failfast bool - debug bool - client *http.Client - sessionVariables map[string]interface{} - // transactions stores transaction timing info. - // key is transaction name, value is map of transaction type and time, e.g. start time and end time. - transactions map[string]map[TransactionType]time.Time - startTime time.Time // record start time of the testcase -} - -// Reset clears runner session variables. -func (r *hrpRunner) Reset() *hrpRunner { - log.Info().Msg("[init] Reset session variables") - r.sessionVariables = make(map[string]interface{}) - r.transactions = make(map[string]map[TransactionType]time.Time) - r.startTime = time.Now() - return r + t *testing.T + failfast bool + debug bool + client *http.Client } // SetFailfast configures whether to stop running when one step fails. @@ -108,43 +92,80 @@ func (r *hrpRunner) Run(testcases ...ITestCase) error { // report execution timing event defer ga.SendEvent(event.StartTiming("execution")) - r.Reset() for _, iTestCase := range testcases { testcase, err := iTestCase.ToTestCase() if err != nil { log.Error().Err(err).Msg("[Run] convert ITestCase interface to TestCase struct failed") return err } - if err := r.runCase(testcase); err != nil { - log.Error().Err(err).Msg("[Run] run testcase failed") + cfg := testcase.Config.ToStruct() + // parse config parameters + err = initParameterIterator(cfg, "runner") + if err != nil { + log.Error().Interface("parameters", cfg.Parameters).Err(err).Msg("parse config parameters failed") return err } + // ๅœจrunnerๆจกๅผไธ‹๏ผŒๆŒ‡ๅฎšๆ•ดไฝ“็ญ–็•ฅ๏ผŒcfg.ParametersSetting.Iteratorsไป…ๅŒ…ๅซไธ€ไธชCartesianProduct็š„่ฟญไปฃๅ™จ + for it := cfg.ParametersSetting.Iterators[0]; it.HasNext(); { + // iterate through all parameter iterators and update case variables + for _, it := range cfg.ParametersSetting.Iterators { + if it.HasNext() { + cfg.Variables = mergeVariables(it.Next(), cfg.Variables) + } + } + if err := r.newCaseRunner(testcase).run(); err != nil { + log.Error().Err(err).Msg("[Run] run testcase failed") + return err + } + } } return nil } -func (r *hrpRunner) runCase(testcase *TestCase) error { - config := testcase.Config +func (r *hrpRunner) newCaseRunner(testcase *TestCase) *caseRunner { + caseRunner := &caseRunner{ + TestCase: testcase, + hrpRunner: r, + } + caseRunner.reset() + return caseRunner +} + +// caseRunner is used to run testcase and its steps. +// each testcase has its own caseRunner instance and share session variables. +type caseRunner struct { + *TestCase + hrpRunner *hrpRunner + sessionVariables map[string]interface{} + // transactions stores transaction timing info. + // key is transaction name, value is map of transaction type and time, e.g. start time and end time. + transactions map[string]map[transactionType]time.Time + startTime time.Time // record start time of the testcase +} + +// reset clears runner session variables. +func (r *caseRunner) reset() *caseRunner { + log.Info().Msg("[init] Reset session variables") + r.sessionVariables = make(map[string]interface{}) + r.transactions = make(map[string]map[transactionType]time.Time) + r.startTime = time.Now() + return r +} + +func (r *caseRunner) run() error { + config := r.TestCase.Config if err := r.parseConfig(config); err != nil { return err } cfg := config.ToStruct() log.Info().Str("testcase", config.Name()).Msg("run testcase start") - parameters := getParameters(config) - if parameters == nil { - parameters = []map[string]interface{}{{}} - } - for _, parameter := range parameters { - cfg.Variables = mergeVariables(parameter, cfg.Variables) - r.startTime = time.Now() - for _, step := range testcase.TestSteps { - _, err := r.runStep(step, config) - if err != nil { - if r.failfast { - log.Error().Err(err).Msg("abort running due to failfast setting") - return err - } - log.Warn().Err(err).Msg("run step failed, continue next step") + + r.startTime = time.Now() + for index := range r.TestCase.TestSteps { + _, err := r.runStep(index, cfg) + if err != nil { + if r.hrpRunner.failfast { + return errors.Wrap(err, "abort running due to failfast setting") } } } @@ -153,7 +174,9 @@ func (r *hrpRunner) runCase(testcase *TestCase) error { return nil } -func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, err error) { +func (r *caseRunner) runStep(index int, caseConfig *TConfig) (stepResult *stepData, err error) { + step := r.TestCase.TestSteps[index] + // step type priority order: transaction > rendezvous > testcase > request if stepTran, ok := step.(*StepTransaction); ok { // transaction step @@ -171,23 +194,18 @@ func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, e log.Error().Err(err).Msg("copy step data failed") return nil, err } - copiedConfig := &TConfig{} - if err = copier.Copy(copiedConfig, config.ToStruct()); err != nil { - log.Error().Err(err).Msg("copy config data failed") - return nil, err - } stepVariables := copiedStep.Variables // override variables // step variables > session variables (extracted variables from previous steps) stepVariables = mergeVariables(stepVariables, r.sessionVariables) // step variables > testcase config variables - stepVariables = mergeVariables(stepVariables, copiedConfig.Variables) + stepVariables = mergeVariables(stepVariables, caseConfig.Variables) // parse step variables parsedVariables, err := parseVariables(stepVariables) if err != nil { - log.Error().Interface("variables", copiedConfig.Variables).Err(err).Msg("parse step variables failed") + log.Error().Interface("variables", caseConfig.Variables).Err(err).Msg("parse step variables failed") return nil, err } copiedStep.Variables = parsedVariables // avoid data racing @@ -204,7 +222,7 @@ func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, e } } else { // run request - copiedStep.Request.URL = buildURL(copiedConfig.BaseURL, copiedStep.Request.URL) // avoid data racing + copiedStep.Request.URL = buildURL(caseConfig.BaseURL, copiedStep.Request.URL) // avoid data racing stepResult, err = r.runStepRequest(copiedStep) if err != nil { log.Error().Err(err).Msg("run request step failed") @@ -225,7 +243,7 @@ func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, e return stepResult, nil } -func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *stepData, err error) { +func (r *caseRunner) runStepTransaction(transaction *Transaction) (stepResult *stepData, err error) { log.Info(). Str("name", transaction.Name). Str("type", string(transaction.Type)). @@ -241,25 +259,25 @@ func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *st // create transaction if not exists if _, ok := r.transactions[transaction.Name]; !ok { - r.transactions[transaction.Name] = make(map[TransactionType]time.Time) + r.transactions[transaction.Name] = make(map[transactionType]time.Time) } // record transaction start time, override if already exists - if transaction.Type == TransactionStart { - r.transactions[transaction.Name][TransactionStart] = time.Now() + if transaction.Type == transactionStart { + r.transactions[transaction.Name][transactionStart] = time.Now() } // record transaction end time, override if already exists - if transaction.Type == TransactionEnd { - r.transactions[transaction.Name][TransactionEnd] = time.Now() + if transaction.Type == transactionEnd { + r.transactions[transaction.Name][transactionEnd] = time.Now() // if transaction start time not exists, use testcase start time instead - if _, ok := r.transactions[transaction.Name][TransactionStart]; !ok { - r.transactions[transaction.Name][TransactionStart] = r.startTime + if _, ok := r.transactions[transaction.Name][transactionStart]; !ok { + r.transactions[transaction.Name][transactionStart] = r.startTime } // calculate transaction duration - duration := r.transactions[transaction.Name][TransactionEnd].Sub( - r.transactions[transaction.Name][TransactionStart]) + duration := r.transactions[transaction.Name][transactionEnd].Sub( + r.transactions[transaction.Name][transactionStart]) stepResult.elapsed = duration.Milliseconds() log.Info().Str("name", transaction.Name).Dur("elapsed", duration).Msg("transaction") } @@ -267,7 +285,7 @@ func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *st return stepResult, nil } -func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, err error) { +func (r *caseRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, err error) { log.Info(). Str("name", rend.Name). Float32("percent", rend.Percent). @@ -282,7 +300,7 @@ func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, e return stepResult, nil } -func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) { +func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) { stepResult = &stepData{ name: step.Name, stepType: stepTypeRequest, @@ -395,7 +413,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error req.Host = u.Host // log & print request - if r.debug { + if r.hrpRunner.debug { reqDump, err := httputil.DumpRequest(req, true) if err != nil { return nil, errors.Wrap(err, "dump request failed") @@ -406,7 +424,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error // do request action start := time.Now() - resp, err := r.client.Do(req) + resp, err := r.hrpRunner.client.Do(req) stepResult.elapsed = time.Since(start).Milliseconds() if err != nil { return nil, errors.Wrap(err, "do request failed") @@ -414,7 +432,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error defer resp.Body.Close() // log & print response - if r.debug { + if r.hrpRunner.debug { fmt.Println("==================== response ===================") respDump, err := httputil.DumpResponse(resp, true) if err != nil { @@ -425,7 +443,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error } // new response object - respObj, err := newResponseObject(r.t, resp) + respObj, err := newResponseObject(r.hrpRunner.t, resp) if err != nil { err = errors.Wrap(err, "init ResponseObject error") return @@ -450,7 +468,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error return stepResult, nil } -func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err error) { +func (r *caseRunner) runStepTestCase(step *TStep) (stepResult *stepData, err error) { stepResult = &stepData{ name: step.Name, stepType: stepTypeTestCase, @@ -458,7 +476,7 @@ func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err erro } testcase := step.TestCase start := time.Now() - err = r.runCase(testcase) + err = r.hrpRunner.newCaseRunner(testcase).run() stepResult.elapsed = time.Since(start).Milliseconds() if err != nil { return stepResult, err @@ -467,7 +485,7 @@ func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err erro return stepResult, nil } -func (r *hrpRunner) parseConfig(config IConfig) error { +func (r *caseRunner) parseConfig(config IConfig) error { cfg := config.ToStruct() // parse config variables parsedVariables, err := parseVariables(cfg.Variables) @@ -493,7 +511,7 @@ func (r *hrpRunner) parseConfig(config IConfig) error { return nil } -func (r *hrpRunner) getSummary() *testCaseSummary { +func (r *caseRunner) getSummary() *testCaseSummary { return &testCaseSummary{} } diff --git a/step.go b/step.go index 4bfcc167..dc7b3bc3 100644 --- a/step.go +++ b/step.go @@ -178,7 +178,7 @@ func (s *StepRequest) CallRefCase(tc *TestCase) *StepTestCaseWithOptionalArgs { func (s *StepRequest) StartTransaction(name string) *StepTransaction { s.step.Transaction = &Transaction{ Name: name, - Type: TransactionStart, + Type: transactionStart, } return &StepTransaction{ step: s.step, @@ -189,7 +189,7 @@ func (s *StepRequest) StartTransaction(name string) *StepTransaction { func (s *StepRequest) EndTransaction(name string) *StepTransaction { s.step.Transaction = &Transaction{ Name: name, - Type: TransactionEnd, + Type: transactionEnd, } return &StepTransaction{ step: s.step, diff --git a/step_test.go b/step_test.go index 38a91abe..d7e3ee17 100644 --- a/step_test.go +++ b/step_test.go @@ -74,12 +74,15 @@ func TestRunRequestPostDataToStruct(t *testing.T) { } func TestRunRequestRun(t *testing.T) { - config := NewConfig("test").SetBaseURL("https://postman-echo.com") - runner := NewRunner(t).SetDebug(true) - if _, err := runner.runStep(stepGET, config); err != nil { + testcase := &TestCase{ + Config: NewConfig("test").SetBaseURL("https://postman-echo.com"), + TestSteps: []IStep{stepGET, stepPOSTData}, + } + runner := NewRunner(t).SetDebug(true).newCaseRunner(testcase) + if _, err := runner.runStep(0, testcase.Config.ToStruct()); err != nil { t.Fatalf("tStep.Run() error: %s", err) } - if _, err := runner.runStep(stepPOSTData, config); err != nil { + if _, err := runner.runStep(1, testcase.Config.ToStruct()); err != nil { t.Fatalf("tStepPOSTData.Run() error: %s", err) } }