diff --git a/README.md b/README.md index ee6ceb5c..cf8e9345 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ - [x] Supports `variables`/`extract`/`validate`/`hooks` mechanisms to create extremely complex test scenarios. - [ ] Built-in integration of rich functions, and you can also use [`go plugin`][plugin] to create and call custom functions. - [x] Inherit all powerful features of [`Boomer`][Boomer] and [`locust`][locust], you can run `load test` without extra work. -- [x] Use it as a `CLI tool` or as a `library` are both supported. +- [x] Using it as a `CLI tool` or a `library` are both supported. See [CHANGELOG]. diff --git a/boomer.go b/boomer.go index 7694f628..97958198 100644 --- a/boomer.go +++ b/boomer.go @@ -62,31 +62,47 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task { Weight: config.Weight, Fn: func() { runner := NewRunner(nil).SetDebug(b.debug).Reset() + + testcaseSuccess := true // flag whole testcase result + var transactionSuccess = true // flag current transaction result + startTime := time.Now() for _, step := range testcase.TestSteps { stepData, err := runner.runStep(step, testcase.Config) - - if stepData.stepType == stepTypeRendezvous { - // TODO: implement rendezvous in boomer - continue - } - - // record transaction - if stepData.stepType == stepTypeTransaction { - if stepData.elapsed != 0 { // only record when transaction ends - b.RecordTransaction(stepData.name, stepData.elapsed, 0) - } - continue - } - - if err == nil { - b.RecordSuccess(step.Type(), step.Name(), stepData.elapsed, stepData.responseLength) - } else { + if err != nil { + // step failed var elapsed int64 if stepData != nil { elapsed = stepData.elapsed } b.RecordFailure(step.Type(), step.Name(), elapsed, err.Error()) + + // update flag + testcaseSuccess = false + transactionSuccess = false + + if runner.failfast { + log.Error().Err(err).Msg("abort running due to failfast setting") + break + } + log.Warn().Err(err).Msg("run step failed, continue next step") + continue + } + + // step success + if stepData.stepType == stepTypeTransaction { + // transaction + // FIXME: support nested transactions + if stepData.elapsed != 0 { // only record when transaction ends + b.RecordTransaction(stepData.name, transactionSuccess, stepData.elapsed, 0) + transactionSuccess = true // reset flag for next transaction + } + } else if stepData.stepType == stepTypeRendezvous { + // rendezvous + // TODO: implement rendezvous in boomer + } else { + // request or testcase step + b.RecordSuccess(step.Type(), step.Name(), stepData.elapsed, stepData.contentSize) } } endTime := time.Now() @@ -96,12 +112,12 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task { if len(transaction) == 1 { // if transaction end time not exists, use testcase end time instead duration := endTime.Sub(transaction[TransactionStart]) - b.RecordTransaction(name, duration.Milliseconds(), 0) + b.RecordTransaction(name, transactionSuccess, duration.Milliseconds(), 0) } } // report testcase as a whole Action transaction, inspired by LoadRunner - b.RecordTransaction("Action", endTime.Sub(startTime).Milliseconds(), 0) + b.RecordTransaction("Action", testcaseSuccess, endTime.Sub(startTime).Milliseconds(), 0) }, } } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cc77f6d9..27ed45b2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,8 +1,9 @@ # Release History -## v0.3.0 (2021-12-13) +## v0.3.0 (2021-12-21) -- feat: implement transaction for load test +- feat: implement transaction mechanism for load test +- feat: support `--continue-on-failure` flag to continue running next step when failure occurs ## v0.2.2 (2021-12-07) diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index cc645658..49cbf6e0 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -22,4 +22,4 @@ Copyright 2021 debugtalk * [hrp har2case](hrp_har2case.md) - Convert HAR to json/yaml testcase files * [hrp run](hrp_run.md) - run API test -###### Auto generated by spf13/cobra on 20-Dec-2021 +###### Auto generated by spf13/cobra on 21-Dec-2021 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index 6181d134..a3171d2c 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -39,4 +39,4 @@ hrp boom [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 20-Dec-2021 +###### Auto generated by spf13/cobra on 21-Dec-2021 diff --git a/docs/cmd/hrp_har2case.md b/docs/cmd/hrp_har2case.md index 46e8b029..cb4c3e30 100644 --- a/docs/cmd/hrp_har2case.md +++ b/docs/cmd/hrp_har2case.md @@ -23,4 +23,4 @@ hrp har2case harPath... [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 20-Dec-2021 +###### Auto generated by spf13/cobra on 21-Dec-2021 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index fa5df0a5..4f68cbed 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -21,13 +21,14 @@ hrp run path... [flags] ### Options ``` - -h, --help help for run - -p, --proxy-url string set proxy url - -s, --silent disable logging request & response details + --continue-on-failure continue running next step when failure occurs + -h, --help help for run + -p, --proxy-url string set proxy url + -s, --silent disable logging request & response details ``` ### SEE ALSO * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 20-Dec-2021 +###### Auto generated by spf13/cobra on 21-Dec-2021 diff --git a/hrp/cmd/run.go b/hrp/cmd/run.go index 6cf20870..42521ac3 100644 --- a/hrp/cmd/run.go +++ b/hrp/cmd/run.go @@ -20,7 +20,7 @@ var runCmd = &cobra.Command{ for _, arg := range args { paths = append(paths, &hrp.TestCasePath{Path: arg}) } - runner := hrp.NewRunner(nil).SetDebug(!silentFlag) + runner := hrp.NewRunner(nil).SetDebug(!silentFlag).SetFailfast(!continueOnFailure) if proxyUrl != "" { runner.SetProxyUrl(proxyUrl) } @@ -29,12 +29,14 @@ var runCmd = &cobra.Command{ } var ( - silentFlag bool - proxyUrl string + continueOnFailure bool + silentFlag bool + proxyUrl string ) func init() { RootCmd.AddCommand(runCmd) + runCmd.Flags().BoolVar(&continueOnFailure, "continue-on-failure", false, "continue running next step when failure occurs") runCmd.Flags().BoolVarP(&silentFlag, "silent", "s", false, "disable logging request & response details") runCmd.Flags().StringVarP(&proxyUrl, "proxy-url", "p", "", "set proxy url") // runCmd.Flags().BoolP("gen-html-report", "r", false, "Generate HTML report") diff --git a/models.go b/models.go index 37724839..cbaf4b4e 100644 --- a/models.go +++ b/models.go @@ -142,10 +142,10 @@ func (tc *TestCase) ToTCase() (*TCase, error) { type testCaseSummary struct{} type stepData struct { - name string // step name - stepType stepType // step type, testcase/request/transaction/rendezvous - success bool // step execution result - elapsed int64 // step execution time in millisecond(ms) - responseLength int64 // response body length - exportVars map[string]interface{} // extract variables + name string // step name + stepType stepType // step type, testcase/request/transaction/rendezvous + success bool // step execution result + elapsed int64 // step execution time in millisecond(ms) + contentSize int64 // response body length + exportVars map[string]interface{} // extract variables } diff --git a/runner.go b/runner.go index b02c59dc..ef7abc20 100644 --- a/runner.go +++ b/runner.go @@ -32,8 +32,9 @@ func NewRunner(t *testing.T) *hrpRunner { t = &testing.T{} } return &hrpRunner{ - t: t, - debug: false, // default to turn off debug + t: t, + failfast: true, // default to failfast + debug: false, // default to turn off debug client: &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -41,11 +42,13 @@ func NewRunner(t *testing.T) *hrpRunner { Timeout: 30 * time.Second, }, sessionVariables: make(map[string]interface{}), + transactions: make(map[string]map[TransactionType]time.Time), } } type hrpRunner struct { t *testing.T + failfast bool debug bool client *http.Client sessionVariables map[string]interface{} @@ -64,6 +67,13 @@ func (r *hrpRunner) Reset() *hrpRunner { return r } +// SetFailfast configures whether to stop running when one step fails. +func (r *hrpRunner) SetFailfast(failfast bool) *hrpRunner { + log.Info().Bool("failfast", failfast).Msg("[init] SetFailfast") + r.failfast = failfast + return r +} + // SetDebug configures whether to log HTTP request and response content. func (r *hrpRunner) SetDebug(debug bool) *hrpRunner { log.Info().Bool("debug", debug).Msg("[init] SetDebug") @@ -123,7 +133,11 @@ func (r *hrpRunner) runCase(testcase *TestCase) error { for _, step := range testcase.TestSteps { _, err := r.runStep(step, config) if err != nil { - return err + if r.failfast { + log.Error().Err(err).Msg("abort running due to failfast setting") + return err + } + log.Warn().Err(err).Msg("run step failed, continue next step") } } @@ -206,11 +220,11 @@ func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *st Msg("transaction") stepResult = &stepData{ - name: transaction.Name, - stepType: stepTypeTransaction, - success: true, - elapsed: 0, - responseLength: 0, // TODO: record transaction total response length + name: transaction.Name, + stepType: stepTypeTransaction, + success: true, + elapsed: 0, + contentSize: 0, // TODO: record transaction total response length } // create transaction if not exists @@ -258,10 +272,10 @@ func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, e func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) { stepResult = &stepData{ - name: step.Name, - stepType: stepTypeRequest, - success: false, - responseLength: 0, + name: step.Name, + stepType: stepTypeRequest, + success: false, + contentSize: 0, } rawUrl := step.Request.URL @@ -420,7 +434,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error } stepResult.success = true - stepResult.responseLength = resp.ContentLength + stepResult.contentSize = resp.ContentLength return stepResult, nil }