Merge pull request #43 from httprunner/refactor-case-runner

v0.3.1

- feat: set ulimit to 10240 before load testing
- fix: concurrent map writes in load testing

boomer.go

@@ -52,19 +52,20 @@ func (b *hrpBoomer) Run(testcases ...ITestCase) {
 }
 
 func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task {
+	hrpRunner := NewRunner(nil).SetDebug(b.debug)
 	config := testcase.Config.ToStruct()
 	return &boomer.Task{
 		Name:   config.Name,
 		Weight: config.Weight,
 		Fn: func() {
-			runner := NewRunner(nil).SetDebug(b.debug).Reset()
+			runner := hrpRunner.newCaseRunner(testcase)
 
 			testcaseSuccess := true       // flag whole testcase result
 			var transactionSuccess = true // flag current transaction result
 
 			startTime := time.Now()
-			for _, step := range testcase.TestSteps {
-				stepData, err := runner.runStep(step, testcase.Config)
+			for index, step := range testcase.TestSteps {
+				stepData, err := runner.runStep(index)
 				if err != nil {
 					// step failed
 					var elapsed int64
@@ -77,8 +78,8 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task {
 					testcaseSuccess = false
 					transactionSuccess = false
 
-					if runner.failfast {
-						log.Error().Err(err).Msg("abort running due to failfast setting")
+					if runner.hrpRunner.failfast {
+						log.Error().Msg("abort running due to failfast setting")
 						break
 					}
 					log.Warn().Err(err).Msg("run step failed, continue next step")
@@ -107,7 +108,7 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task {
 			for name, transaction := range runner.transactions {
 				if len(transaction) == 1 {
 					// if transaction end time not exists, use testcase end time instead
-					duration := endTime.Sub(transaction[TransactionStart])
+					duration := endTime.Sub(transaction[transactionStart])
 					b.RecordTransaction(name, transactionSuccess, duration.Milliseconds(), 0)
 				}
 			}

docs/CHANGELOG.md

@@ -1,15 +1,19 @@
 # Release History
 
-## v0.3.0 (2021-12-22)
+## v0.3.1 (2021-12-30)
+
+- feat: set ulimit to 10240 before load testing
+- fix: concurrent map writes in load testing
+
+## v0.3.0 (2021-12-24)
 
 - feat: implement `transaction` mechanism for load test
 - feat: continue running next step when failure occurs with `--continue-on-failure` flag, default to failfast
-- feat: spawn workers with `--spawn-rate` flag
-- refactor: fork [boomer] as sub module
 - feat: report GA events with version
-- feat: run load test with the given limit and burst as rate limiter
+- feat: run load test with the given limit and burst as rate limiter, use `--spawn-count`, `--spawn-rate` and `--request-increase-rate` flag
+- feat: report runner state to prometheus
+- refactor: fork [boomer] as submodule initially and made a lot of changes
 - change: update API models
-- feat: report runner state
 
 ## v0.2.2 (2021-12-07)
 

docs/cmd/hrp.md

@@ -4,9 +4,19 @@ One-stop solution for HTTP(S) testing.
 
 ### Synopsis
 
-hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨
+
+██╗  ██╗████████╗████████╗██████╗ ██████╗ ██╗   ██╗███╗   ██╗███╗   ██╗███████╗██████╗
+██║  ██║╚══██╔══╝╚══██╔══╝██╔══██╗██╔══██╗██║   ██║████╗  ██║████╗  ██║██╔════╝██╔══██╗
+███████║   ██║      ██║   ██████╔╝██████╔╝██║   ██║██╔██╗ ██║██╔██╗ ██║█████╗  ██████╔╝
+██╔══██║   ██║      ██║   ██╔═══╝ ██╔══██╗██║   ██║██║╚██╗██║██║╚██╗██║██╔══╝  ██╔══██╗
+██║  ██║   ██║      ██║   ██║     ██║  ██║╚██████╔╝██║ ╚████║██║ ╚████║███████╗██║  ██║
+╚═╝  ╚═╝   ╚═╝      ╚═╝   ╚═╝     ╚═╝  ╚═╝ ╚═════╝ ╚═╝  ╚═══╝╚═╝  ╚═══╝╚══════╝╚═╝  ╚═╝
+
+hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing,
+load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨
 
 License: Apache-2.0
+Website: https://httprunner.com
 Github: https://github.com/httprunner/hrp
 Copyright 2021 debugtalk
 
@@ -22,4 +32,4 @@ Copyright 2021 debugtalk
 * [hrp har2case](hrp_har2case.md)	 - Convert HAR to json/yaml testcase files
 * [hrp run](hrp_run.md)	 - run API test
 
-###### Auto generated by spf13/cobra on 24-Dec-2021
+###### Auto generated by spf13/cobra on 30-Dec-2021

docs/cmd/hrp_boom.md

@@ -38,4 +38,4 @@ hrp boom [flags]
 
 * [hrp](hrp.md)	 - One-stop solution for HTTP(S) testing.
 
-###### Auto generated by spf13/cobra on 24-Dec-2021
+###### Auto generated by spf13/cobra on 30-Dec-2021

docs/cmd/hrp_har2case.md

@@ -23,4 +23,4 @@ hrp har2case harPath... [flags]
 
 * [hrp](hrp.md)	 - One-stop solution for HTTP(S) testing.
 
-###### Auto generated by spf13/cobra on 24-Dec-2021
+###### Auto generated by spf13/cobra on 30-Dec-2021

docs/cmd/hrp_run.md

@@ -31,4 +31,4 @@ hrp run path... [flags]
 
 * [hrp](hrp.md)	 - One-stop solution for HTTP(S) testing.
 
-###### Auto generated by spf13/cobra on 24-Dec-2021
+###### Auto generated by spf13/cobra on 30-Dec-2021

hrp/cmd/boom.go

@@ -19,7 +19,8 @@ var boomCmd = &cobra.Command{
   $ hrp boom examples/	# run testcases in specified folder`,
 	Args: cobra.MinimumNArgs(1),
 	PreRun: func(cmd *cobra.Command, args []string) {
-		setLogLevel("WARN") // disable info logs for load testing
+		boomer.SetUlimit(10240) // ulimit -n 10240
+		setLogLevel("WARN")     // disable info logs for load testing
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		var paths []hrp.ITestCase

hrp/cmd/har2case.go

@@ -4,7 +4,7 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
-	"github.com/httprunner/hrp/har2case"
+	"github.com/httprunner/hrp/internal/har2case"
 )
 
 // har2caseCmd represents the har2case command

hrp/cmd/root.go

@@ -15,9 +15,19 @@ import (
 var RootCmd = &cobra.Command{
 	Use:   "hrp",
 	Short: "One-stop solution for HTTP(S) testing.",
-	Long: `hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨
+	Long: `
+██╗  ██╗████████╗████████╗██████╗ ██████╗ ██╗   ██╗███╗   ██╗███╗   ██╗███████╗██████╗
+██║  ██║╚══██╔══╝╚══██╔══╝██╔══██╗██╔══██╗██║   ██║████╗  ██║████╗  ██║██╔════╝██╔══██╗
+███████║   ██║      ██║   ██████╔╝██████╔╝██║   ██║██╔██╗ ██║██╔██╗ ██║█████╗  ██████╔╝
+██╔══██║   ██║      ██║   ██╔═══╝ ██╔══██╗██║   ██║██║╚██╗██║██║╚██╗██║██╔══╝  ██╔══██╗
+██║  ██║   ██║      ██║   ██║     ██║  ██║╚██████╔╝██║ ╚████║██║ ╚████║███████╗██║  ██║
+╚═╝  ╚═╝   ╚═╝      ╚═╝   ╚═╝     ╚═╝  ╚═╝ ╚═════╝ ╚═╝  ╚═══╝╚═╝  ╚═══╝╚══════╝╚═╝  ╚═╝
+
+hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing,
+load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨
 
 License: Apache-2.0
+Website: https://httprunner.com
 Github: https://github.com/httprunner/hrp
 Copyright 2021 debugtalk`,
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {

internal/boomer/ulimit.go

@@ -0,0 +1,32 @@
+// +build !windows
+
+package boomer
+
+import (
+	"syscall"
+
+	"github.com/rs/zerolog/log"
+)
+
+// set resource limit
+// ulimit -n 10240
+func SetUlimit(limit uint64) {
+	var rLimit syscall.Rlimit
+	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		log.Error().Err(err).Msg("get ulimit failed")
+		return
+	}
+	log.Info().Uint64("limit", rLimit.Cur).Msg("get current ulimit")
+	if rLimit.Cur >= limit {
+		return
+	}
+
+	rLimit.Cur = limit
+	log.Info().Uint64("limit", rLimit.Cur).Msg("set current ulimit")
+	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		log.Error().Err(err).Msg("set ulimit failed")
+		return
+	}
+}

internal/boomer/ulimit_windows.go

@@ -0,0 +1,12 @@
+// +build windows
+
+package boomer
+
+import (
+	"github.com/rs/zerolog/log"
+)
+
+// set resource limit
+func SetUlimit(limit uint64) {
+	log.Warn().Msg("windows does not support setting ulimit")
+}

internal/har2case/core_test.go

@@ -7,8 +7,8 @@ import (
 )
 
 var (
-	harPath  = "../examples/har/demo.har"
-	harPath2 = "../examples/har/postman-echo.har"
+	harPath  = "../../examples/har/demo.har"
+	harPath2 = "../../examples/har/postman-echo.har"
 )
 
 func TestGenJSON(t *testing.T) {

internal/version/init.go

@@ -1,3 +1,3 @@
 package version
 
-const VERSION = "v0.3.0"
+const VERSION = "v0.4.0"

models.go

@@ -69,16 +69,16 @@ const (
 	stepTypeRendezvous  stepType = "rendezvous"
 )
 
-type TransactionType string
+type transactionType string
 
 const (
-	TransactionStart TransactionType = "start"
-	TransactionEnd   TransactionType = "end"
+	transactionStart transactionType = "start"
+	transactionEnd   transactionType = "end"
 )
 
 type Transaction struct {
 	Name string          `json:"name" yaml:"name"`
-	Type TransactionType `json:"type" yaml:"type"`
+	Type transactionType `json:"type" yaml:"type"`
 }
 type Rendezvous struct {
 	Name    string  `json:"name" yaml:"name"`                           // required

runner.go

@@ -42,30 +42,14 @@ func NewRunner(t *testing.T) *hrpRunner {
 			},
 			Timeout: 30 * time.Second,
 		},
-		sessionVariables: make(map[string]interface{}),
-		transactions:     make(map[string]map[TransactionType]time.Time),
 	}
 }
 
 type hrpRunner struct {
-	t                *testing.T
-	failfast         bool
-	debug            bool
-	client           *http.Client
-	sessionVariables map[string]interface{}
-	// transactions stores transaction timing info.
-	// key is transaction name, value is map of transaction type and time, e.g. start time and end time.
-	transactions map[string]map[TransactionType]time.Time
-	startTime    time.Time // record start time of the testcase
-}
-
-// Reset clears runner session variables.
-func (r *hrpRunner) Reset() *hrpRunner {
-	log.Info().Msg("[init] Reset session variables")
-	r.sessionVariables = make(map[string]interface{})
-	r.transactions = make(map[string]map[TransactionType]time.Time)
-	r.startTime = time.Now()
-	return r
+	t        *testing.T
+	failfast bool
+	debug    bool
+	client   *http.Client
 }
 
 // SetFailfast configures whether to stop running when one step fails.
@@ -108,14 +92,13 @@ func (r *hrpRunner) Run(testcases ...ITestCase) error {
 	// report execution timing event
 	defer ga.SendEvent(event.StartTiming("execution"))
 
-	r.Reset()
 	for _, iTestCase := range testcases {
 		testcase, err := iTestCase.ToTestCase()
 		if err != nil {
 			log.Error().Err(err).Msg("[Run] convert ITestCase interface to TestCase struct failed")
 			return err
 		}
-		if err := r.runCase(testcase); err != nil {
+		if err := r.newCaseRunner(testcase).run(); err != nil {
 			log.Error().Err(err).Msg("[Run] run testcase failed")
 			return err
 		}
@@ -123,20 +106,49 @@ func (r *hrpRunner) Run(testcases ...ITestCase) error {
 	return nil
 }
 
-func (r *hrpRunner) runCase(testcase *TestCase) error {
-	config := testcase.Config
+func (r *hrpRunner) newCaseRunner(testcase *TestCase) *caseRunner {
+	caseRunner := &caseRunner{
+		TestCase:  testcase,
+		hrpRunner: r,
+	}
+	caseRunner.reset()
+	return caseRunner
+}
+
+// caseRunner is used to run testcase and its steps.
+// each testcase has its own caseRunner instance and share session variables.
+type caseRunner struct {
+	*TestCase
+	hrpRunner        *hrpRunner
+	sessionVariables map[string]interface{}
+	// transactions stores transaction timing info.
+	// key is transaction name, value is map of transaction type and time, e.g. start time and end time.
+	transactions map[string]map[transactionType]time.Time
+	startTime    time.Time // record start time of the testcase
+}
+
+// reset clears runner session variables.
+func (r *caseRunner) reset() *caseRunner {
+	log.Info().Msg("[init] Reset session variables")
+	r.sessionVariables = make(map[string]interface{})
+	r.transactions = make(map[string]map[transactionType]time.Time)
+	r.startTime = time.Now()
+	return r
+}
+
+func (r *caseRunner) run() error {
+	config := r.TestCase.Config
 	if err := r.parseConfig(config); err != nil {
 		return err
 	}
 
 	log.Info().Str("testcase", config.Name()).Msg("run testcase start")
 	r.startTime = time.Now()
-	for _, step := range testcase.TestSteps {
-		_, err := r.runStep(step, config)
+	for index := range r.TestCase.TestSteps {
+		_, err := r.runStep(index)
 		if err != nil {
-			if r.failfast {
-				log.Error().Err(err).Msg("abort running due to failfast setting")
-				return err
+			if r.hrpRunner.failfast {
+				return errors.Wrap(err, "abort running due to failfast setting")
 			}
 			log.Warn().Err(err).Msg("run step failed, continue next step")
 		}
@@ -146,7 +158,10 @@ func (r *hrpRunner) runCase(testcase *TestCase) error {
 	return nil
 }
 
-func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, err error) {
+func (r *caseRunner) runStep(index int) (stepResult *stepData, err error) {
+	config := r.TestCase.Config
+	step := r.TestCase.TestSteps[index]
+
 	// step type priority order: transaction > rendezvous > testcase > request
 	if stepTran, ok := step.(*StepTransaction); ok {
 		// transaction step
@@ -218,7 +233,7 @@ func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, e
 	return stepResult, nil
 }
 
-func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *stepData, err error) {
+func (r *caseRunner) runStepTransaction(transaction *Transaction) (stepResult *stepData, err error) {
 	log.Info().
 		Str("name", transaction.Name).
 		Str("type", string(transaction.Type)).
@@ -234,25 +249,25 @@ func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *st
 
 	// create transaction if not exists
 	if _, ok := r.transactions[transaction.Name]; !ok {
-		r.transactions[transaction.Name] = make(map[TransactionType]time.Time)
+		r.transactions[transaction.Name] = make(map[transactionType]time.Time)
 	}
 
 	// record transaction start time, override if already exists
-	if transaction.Type == TransactionStart {
-		r.transactions[transaction.Name][TransactionStart] = time.Now()
+	if transaction.Type == transactionStart {
+		r.transactions[transaction.Name][transactionStart] = time.Now()
 	}
 	// record transaction end time, override if already exists
-	if transaction.Type == TransactionEnd {
-		r.transactions[transaction.Name][TransactionEnd] = time.Now()
+	if transaction.Type == transactionEnd {
+		r.transactions[transaction.Name][transactionEnd] = time.Now()
 
 		// if transaction start time not exists, use testcase start time instead
-		if _, ok := r.transactions[transaction.Name][TransactionStart]; !ok {
-			r.transactions[transaction.Name][TransactionStart] = r.startTime
+		if _, ok := r.transactions[transaction.Name][transactionStart]; !ok {
+			r.transactions[transaction.Name][transactionStart] = r.startTime
 		}
 
 		// calculate transaction duration
-		duration := r.transactions[transaction.Name][TransactionEnd].Sub(
-			r.transactions[transaction.Name][TransactionStart])
+		duration := r.transactions[transaction.Name][transactionEnd].Sub(
+			r.transactions[transaction.Name][transactionStart])
 		stepResult.elapsed = duration.Milliseconds()
 		log.Info().Str("name", transaction.Name).Dur("elapsed", duration).Msg("transaction")
 	}
@@ -260,7 +275,7 @@ func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *st
 	return stepResult, nil
 }
 
-func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, err error) {
+func (r *caseRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, err error) {
 	log.Info().
 		Str("name", rend.Name).
 		Float32("percent", rend.Percent).
@@ -275,7 +290,7 @@ func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, e
 	return stepResult, nil
 }
 
-func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) {
+func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) {
 	stepResult = &stepData{
 		name:        step.Name,
 		stepType:    stepTypeRequest,
@@ -388,7 +403,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error
 	req.Host = u.Host
 
 	// log & print request
-	if r.debug {
+	if r.hrpRunner.debug {
 		reqDump, err := httputil.DumpRequest(req, true)
 		if err != nil {
 			return nil, errors.Wrap(err, "dump request failed")
@@ -399,7 +414,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error
 
 	// do request action
 	start := time.Now()
-	resp, err := r.client.Do(req)
+	resp, err := r.hrpRunner.client.Do(req)
 	stepResult.elapsed = time.Since(start).Milliseconds()
 	if err != nil {
 		return nil, errors.Wrap(err, "do request failed")
@@ -407,7 +422,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error
 	defer resp.Body.Close()
 
 	// log & print response
-	if r.debug {
+	if r.hrpRunner.debug {
 		fmt.Println("==================== response ===================")
 		respDump, err := httputil.DumpResponse(resp, true)
 		if err != nil {
@@ -418,7 +433,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error
 	}
 
 	// new response object
-	respObj, err := newResponseObject(r.t, resp)
+	respObj, err := newResponseObject(r.hrpRunner.t, resp)
 	if err != nil {
 		err = errors.Wrap(err, "init ResponseObject error")
 		return
@@ -443,7 +458,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error
 	return stepResult, nil
 }
 
-func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err error) {
+func (r *caseRunner) runStepTestCase(step *TStep) (stepResult *stepData, err error) {
 	stepResult = &stepData{
 		name:     step.Name,
 		stepType: stepTypeTestCase,
@@ -451,7 +466,7 @@ func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err erro
 	}
 	testcase := step.TestCase
 	start := time.Now()
-	err = r.runCase(testcase)
+	err = r.hrpRunner.newCaseRunner(testcase).run()
 	stepResult.elapsed = time.Since(start).Milliseconds()
 	if err != nil {
 		return stepResult, err
@@ -460,7 +475,7 @@ func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err erro
 	return stepResult, nil
 }
 
-func (r *hrpRunner) parseConfig(config IConfig) error {
+func (r *caseRunner) parseConfig(config IConfig) error {
 	cfg := config.ToStruct()
 	// parse config variables
 	parsedVariables, err := parseVariables(cfg.Variables)
@@ -487,7 +502,7 @@ func (r *hrpRunner) parseConfig(config IConfig) error {
 	return nil
 }
 
-func (r *hrpRunner) getSummary() *testCaseSummary {
+func (r *caseRunner) getSummary() *testCaseSummary {
 	return &testCaseSummary{}
 }
 

step.go

@@ -178,7 +178,7 @@ func (s *StepRequest) CallRefCase(tc *TestCase) *StepTestCaseWithOptionalArgs {
 func (s *StepRequest) StartTransaction(name string) *StepTransaction {
 	s.step.Transaction = &Transaction{
 		Name: name,
-		Type: TransactionStart,
+		Type: transactionStart,
 	}
 	return &StepTransaction{
 		step: s.step,
@@ -189,7 +189,7 @@ func (s *StepRequest) StartTransaction(name string) *StepTransaction {
 func (s *StepRequest) EndTransaction(name string) *StepTransaction {
 	s.step.Transaction = &Transaction{
 		Name: name,
-		Type: TransactionEnd,
+		Type: transactionEnd,
 	}
 	return &StepTransaction{
 		step: s.step,

step_test.go

@@ -74,12 +74,15 @@ func TestRunRequestPostDataToStruct(t *testing.T) {
 }
 
 func TestRunRequestRun(t *testing.T) {
-	config := NewConfig("test").SetBaseURL("https://postman-echo.com")
-	runner := NewRunner(t).SetDebug(true)
-	if _, err := runner.runStep(stepGET, config); err != nil {
+	testcase := &TestCase{
+		Config:    NewConfig("test").SetBaseURL("https://postman-echo.com"),
+		TestSteps: []IStep{stepGET, stepPOSTData},
+	}
+	runner := NewRunner(t).SetDebug(true).newCaseRunner(testcase)
+	if _, err := runner.runStep(0); err != nil {
 		t.Fatalf("tStep.Run() error: %s", err)
 	}
-	if _, err := runner.runStep(stepPOSTData, config); err != nil {
+	if _, err := runner.runStep(1); err != nil {
 		t.Fatalf("tStepPOSTData.Run() error: %s", err)
 	}
 }