fix: report transaction result

README.md

@@ -18,7 +18,7 @@
 - [x] Supports `variables`/`extract`/`validate`/`hooks` mechanisms to create extremely complex test scenarios.
 - [ ] Built-in integration of rich functions, and you can also use [`go plugin`][plugin] to create and call custom functions.
 - [x] Inherit all powerful features of [`Boomer`][Boomer] and [`locust`][locust], you can run `load test` without extra work.
-- [x] Use it as a `CLI tool` or as a `library` are both supported.
+- [x] Using it as a `CLI tool` or a `library` are both supported.
 
 See [CHANGELOG].
 

boomer.go

@@ -62,31 +62,47 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task {
 		Weight: config.Weight,
 		Fn: func() {
 			runner := NewRunner(nil).SetDebug(b.debug).Reset()
+
+			testcaseSuccess := true       // flag whole testcase result
+			var transactionSuccess = true // flag current transaction result
+
 			startTime := time.Now()
 			for _, step := range testcase.TestSteps {
 				stepData, err := runner.runStep(step, testcase.Config)
-
-				if stepData.stepType == stepTypeRendezvous {
-					// TODO: implement rendezvous in boomer
-					continue
-				}
-
-				// record transaction
-				if stepData.stepType == stepTypeTransaction {
-					if stepData.elapsed != 0 { // only record when transaction ends
-						b.RecordTransaction(stepData.name, stepData.elapsed, 0)
-					}
-					continue
-				}
-
-				if err == nil {
-					b.RecordSuccess(step.Type(), step.Name(), stepData.elapsed, stepData.responseLength)
-				} else {
+				if err != nil {
+					// step failed
 					var elapsed int64
 					if stepData != nil {
 						elapsed = stepData.elapsed
 					}
 					b.RecordFailure(step.Type(), step.Name(), elapsed, err.Error())
+
+					// update flag
+					testcaseSuccess = false
+					transactionSuccess = false
+
+					if runner.failfast {
+						log.Error().Err(err).Msg("abort running due to failfast setting")
+						break
+					}
+					log.Warn().Err(err).Msg("run step failed, continue next step")
+					continue
+				}
+
+				// step success
+				if stepData.stepType == stepTypeTransaction {
+					// transaction
+					// FIXME: support nested transactions
+					if stepData.elapsed != 0 { // only record when transaction ends
+						b.RecordTransaction(stepData.name, transactionSuccess, stepData.elapsed, 0)
+						transactionSuccess = true // reset flag for next transaction
+					}
+				} else if stepData.stepType == stepTypeRendezvous {
+					// rendezvous
+					// TODO: implement rendezvous in boomer
+				} else {
+					// request or testcase step
+					b.RecordSuccess(step.Type(), step.Name(), stepData.elapsed, stepData.contentSize)
 				}
 			}
 			endTime := time.Now()
@@ -96,12 +112,12 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task {
 				if len(transaction) == 1 {
 					// if transaction end time not exists, use testcase end time instead
 					duration := endTime.Sub(transaction[TransactionStart])
-					b.RecordTransaction(name, duration.Milliseconds(), 0)
+					b.RecordTransaction(name, transactionSuccess, duration.Milliseconds(), 0)
 				}
 			}
 
 			// report testcase as a whole Action transaction, inspired by LoadRunner
-			b.RecordTransaction("Action", endTime.Sub(startTime).Milliseconds(), 0)
+			b.RecordTransaction("Action", testcaseSuccess, endTime.Sub(startTime).Milliseconds(), 0)
 		},
 	}
 }

docs/CHANGELOG.md

@@ -1,8 +1,9 @@
 # Release History
 
-## v0.3.0 (2021-12-13)
+## v0.3.0 (2021-12-21)
 
-- feat: implement transaction for load test
+- feat: implement transaction mechanism for load test
+- feat: support `--continue-on-failure` flag to continue running next step when failure occurs
 
 ## v0.2.2 (2021-12-07)
 

docs/cmd/hrp.md

@@ -22,4 +22,4 @@ Copyright 2021 debugtalk
 * [hrp har2case](hrp_har2case.md)	 - Convert HAR to json/yaml testcase files
 * [hrp run](hrp_run.md)	 - run API test
 
-###### Auto generated by spf13/cobra on 20-Dec-2021
+###### Auto generated by spf13/cobra on 21-Dec-2021

docs/cmd/hrp_boom.md

@@ -39,4 +39,4 @@ hrp boom [flags]
 
 * [hrp](hrp.md)	 - One-stop solution for HTTP(S) testing.
 
-###### Auto generated by spf13/cobra on 20-Dec-2021
+###### Auto generated by spf13/cobra on 21-Dec-2021

docs/cmd/hrp_har2case.md

@@ -23,4 +23,4 @@ hrp har2case harPath... [flags]
 
 * [hrp](hrp.md)	 - One-stop solution for HTTP(S) testing.
 
-###### Auto generated by spf13/cobra on 20-Dec-2021
+###### Auto generated by spf13/cobra on 21-Dec-2021

docs/cmd/hrp_run.md

@@ -21,13 +21,14 @@ hrp run path... [flags]
 ### Options
 
 ```
-  -h, --help               help for run
-  -p, --proxy-url string   set proxy url
-  -s, --silent             disable logging request & response details
+      --continue-on-failure   continue running next step when failure occurs
+  -h, --help                  help for run
+  -p, --proxy-url string      set proxy url
+  -s, --silent                disable logging request & response details
 ```
 
 ### SEE ALSO
 
 * [hrp](hrp.md)	 - One-stop solution for HTTP(S) testing.
 
-###### Auto generated by spf13/cobra on 20-Dec-2021
+###### Auto generated by spf13/cobra on 21-Dec-2021

hrp/cmd/run.go

@@ -20,7 +20,7 @@ var runCmd = &cobra.Command{
 		for _, arg := range args {
 			paths = append(paths, &hrp.TestCasePath{Path: arg})
 		}
-		runner := hrp.NewRunner(nil).SetDebug(!silentFlag)
+		runner := hrp.NewRunner(nil).SetDebug(!silentFlag).SetFailfast(!continueOnFailure)
 		if proxyUrl != "" {
 			runner.SetProxyUrl(proxyUrl)
 		}
@@ -29,12 +29,14 @@ var runCmd = &cobra.Command{
 }
 
 var (
-	silentFlag bool
-	proxyUrl   string
+	continueOnFailure bool
+	silentFlag        bool
+	proxyUrl          string
 )
 
 func init() {
 	RootCmd.AddCommand(runCmd)
+	runCmd.Flags().BoolVar(&continueOnFailure, "continue-on-failure", false, "continue running next step when failure occurs")
 	runCmd.Flags().BoolVarP(&silentFlag, "silent", "s", false, "disable logging request & response details")
 	runCmd.Flags().StringVarP(&proxyUrl, "proxy-url", "p", "", "set proxy url")
 	// runCmd.Flags().BoolP("gen-html-report", "r", false, "Generate HTML report")

models.go

@@ -142,10 +142,10 @@ func (tc *TestCase) ToTCase() (*TCase, error) {
 type testCaseSummary struct{}
 
 type stepData struct {
-	name           string                 // step name
-	stepType       stepType               // step type, testcase/request/transaction/rendezvous
-	success        bool                   // step execution result
-	elapsed        int64                  // step execution time in millisecond(ms)
-	responseLength int64                  // response body length
-	exportVars     map[string]interface{} // extract variables
+	name        string                 // step name
+	stepType    stepType               // step type, testcase/request/transaction/rendezvous
+	success     bool                   // step execution result
+	elapsed     int64                  // step execution time in millisecond(ms)
+	contentSize int64                  // response body length
+	exportVars  map[string]interface{} // extract variables
 }

runner.go

@@ -32,8 +32,9 @@ func NewRunner(t *testing.T) *hrpRunner {
 		t = &testing.T{}
 	}
 	return &hrpRunner{
-		t:     t,
-		debug: false, // default to turn off debug
+		t:        t,
+		failfast: true,  // default to failfast
+		debug:    false, // default to turn off debug
 		client: &http.Client{
 			Transport: &http.Transport{
 				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
@@ -41,11 +42,13 @@ func NewRunner(t *testing.T) *hrpRunner {
 			Timeout: 30 * time.Second,
 		},
 		sessionVariables: make(map[string]interface{}),
+		transactions:     make(map[string]map[TransactionType]time.Time),
 	}
 }
 
 type hrpRunner struct {
 	t                *testing.T
+	failfast         bool
 	debug            bool
 	client           *http.Client
 	sessionVariables map[string]interface{}
@@ -64,6 +67,13 @@ func (r *hrpRunner) Reset() *hrpRunner {
 	return r
 }
 
+// SetFailfast configures whether to stop running when one step fails.
+func (r *hrpRunner) SetFailfast(failfast bool) *hrpRunner {
+	log.Info().Bool("failfast", failfast).Msg("[init] SetFailfast")
+	r.failfast = failfast
+	return r
+}
+
 // SetDebug configures whether to log HTTP request and response content.
 func (r *hrpRunner) SetDebug(debug bool) *hrpRunner {
 	log.Info().Bool("debug", debug).Msg("[init] SetDebug")
@@ -123,7 +133,11 @@ func (r *hrpRunner) runCase(testcase *TestCase) error {
 	for _, step := range testcase.TestSteps {
 		_, err := r.runStep(step, config)
 		if err != nil {
-			return err
+			if r.failfast {
+				log.Error().Err(err).Msg("abort running due to failfast setting")
+				return err
+			}
+			log.Warn().Err(err).Msg("run step failed, continue next step")
 		}
 	}
 
@@ -206,11 +220,11 @@ func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *st
 		Msg("transaction")
 
 	stepResult = &stepData{
-		name:           transaction.Name,
-		stepType:       stepTypeTransaction,
-		success:        true,
-		elapsed:        0,
-		responseLength: 0, // TODO: record transaction total response length
+		name:        transaction.Name,
+		stepType:    stepTypeTransaction,
+		success:     true,
+		elapsed:     0,
+		contentSize: 0, // TODO: record transaction total response length
 	}
 
 	// create transaction if not exists
@@ -258,10 +272,10 @@ func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, e
 
 func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) {
 	stepResult = &stepData{
-		name:           step.Name,
-		stepType:       stepTypeRequest,
-		success:        false,
-		responseLength: 0,
+		name:        step.Name,
+		stepType:    stepTypeRequest,
+		success:     false,
+		contentSize: 0,
 	}
 
 	rawUrl := step.Request.URL
@@ -420,7 +434,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error
 	}
 
 	stepResult.success = true
-	stepResult.responseLength = resp.ContentLength
+	stepResult.contentSize = resp.ContentLength
 	return stepResult, nil
 }