diff --git a/boomer.go b/boomer.go index fb8b5d71..eda4a4f2 100644 --- a/boomer.go +++ b/boomer.go @@ -52,19 +52,20 @@ func (b *hrpBoomer) Run(testcases ...ITestCase) { } func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task { + hrpRunner := NewRunner(nil).SetDebug(b.debug) config := testcase.Config.ToStruct() return &boomer.Task{ Name: config.Name, Weight: config.Weight, Fn: func() { - runner := NewRunner(nil).SetDebug(b.debug).Reset() + runner := hrpRunner.newCaseRunner(testcase) testcaseSuccess := true // flag whole testcase result var transactionSuccess = true // flag current transaction result startTime := time.Now() - for _, step := range testcase.TestSteps { - stepData, err := runner.runStep(step, testcase.Config) + for index, step := range testcase.TestSteps { + stepData, err := runner.runStep(index) if err != nil { // step failed var elapsed int64 @@ -77,8 +78,8 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task { testcaseSuccess = false transactionSuccess = false - if runner.failfast { - log.Error().Err(err).Msg("abort running due to failfast setting") + if runner.hrpRunner.failfast { + log.Error().Msg("abort running due to failfast setting") break } log.Warn().Err(err).Msg("run step failed, continue next step") @@ -107,7 +108,7 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task { for name, transaction := range runner.transactions { if len(transaction) == 1 { // if transaction end time not exists, use testcase end time instead - duration := endTime.Sub(transaction[TransactionStart]) + duration := endTime.Sub(transaction[transactionStart]) b.RecordTransaction(name, transactionSuccess, duration.Milliseconds(), 0) } } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 67354670..4f2f8207 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,15 +1,19 @@ # Release History -## v0.3.0 (2021-12-22) +## v0.3.1 (2021-12-30) + +- feat: set ulimit to 10240 before load testing +- fix: concurrent map writes in load testing + +## v0.3.0 (2021-12-24) - feat: implement `transaction` mechanism for load test - feat: continue running next step when failure occurs with `--continue-on-failure` flag, default to failfast -- feat: spawn workers with `--spawn-rate` flag -- refactor: fork [boomer] as sub module - feat: report GA events with version -- feat: run load test with the given limit and burst as rate limiter +- feat: run load test with the given limit and burst as rate limiter, use `--spawn-count`, `--spawn-rate` and `--request-increase-rate` flag +- feat: report runner state to prometheus +- refactor: fork [boomer] as submodule initially and made a lot of changes - change: update API models -- feat: report runner state ## v0.2.2 (2021-12-07) diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index b2067d4a..8b751a2d 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -4,9 +4,19 @@ One-stop solution for HTTP(S) testing. ### Synopsis -hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨ + +██╗ ██╗████████╗████████╗██████╗ ██████╗ ██╗ ██╗███╗ ██╗███╗ ██╗███████╗██████╗ +██║ ██║╚══██╔══╝╚══██╔══╝██╔══██╗██╔══██╗██║ ██║████╗ ██║████╗ ██║██╔════╝██╔══██╗ +███████║ ██║ ██║ ██████╔╝██████╔╝██║ ██║██╔██╗ ██║██╔██╗ ██║█████╗ ██████╔╝ +██╔══██║ ██║ ██║ ██╔═══╝ ██╔══██╗██║ ██║██║╚██╗██║██║╚██╗██║██╔══╝ ██╔══██╗ +██║ ██║ ██║ ██║ ██║ ██║ ██║╚██████╔╝██║ ╚████║██║ ╚████║███████╗██║ ██║ +╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝╚═╝ ╚═══╝╚══════╝╚═╝ ╚═╝ + +hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, +load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨ License: Apache-2.0 +Website: https://httprunner.com Github: https://github.com/httprunner/hrp Copyright 2021 debugtalk @@ -22,4 +32,4 @@ Copyright 2021 debugtalk * [hrp har2case](hrp_har2case.md) - Convert HAR to json/yaml testcase files * [hrp run](hrp_run.md) - run API test -###### Auto generated by spf13/cobra on 24-Dec-2021 +###### Auto generated by spf13/cobra on 30-Dec-2021 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index c5a65c42..494ae8d6 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -38,4 +38,4 @@ hrp boom [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 24-Dec-2021 +###### Auto generated by spf13/cobra on 30-Dec-2021 diff --git a/docs/cmd/hrp_har2case.md b/docs/cmd/hrp_har2case.md index 8e559e4b..d9664517 100644 --- a/docs/cmd/hrp_har2case.md +++ b/docs/cmd/hrp_har2case.md @@ -23,4 +23,4 @@ hrp har2case harPath... [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 24-Dec-2021 +###### Auto generated by spf13/cobra on 30-Dec-2021 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index 96cd3590..9cb28dbf 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -31,4 +31,4 @@ hrp run path... [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 24-Dec-2021 +###### Auto generated by spf13/cobra on 30-Dec-2021 diff --git a/hrp/cmd/boom.go b/hrp/cmd/boom.go index 41c67758..b19c239a 100644 --- a/hrp/cmd/boom.go +++ b/hrp/cmd/boom.go @@ -19,7 +19,8 @@ var boomCmd = &cobra.Command{ $ hrp boom examples/ # run testcases in specified folder`, Args: cobra.MinimumNArgs(1), PreRun: func(cmd *cobra.Command, args []string) { - setLogLevel("WARN") // disable info logs for load testing + boomer.SetUlimit(10240) // ulimit -n 10240 + setLogLevel("WARN") // disable info logs for load testing }, Run: func(cmd *cobra.Command, args []string) { var paths []hrp.ITestCase diff --git a/hrp/cmd/har2case.go b/hrp/cmd/har2case.go index 73670a57..7512f241 100644 --- a/hrp/cmd/har2case.go +++ b/hrp/cmd/har2case.go @@ -4,7 +4,7 @@ import ( "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "github.com/httprunner/hrp/har2case" + "github.com/httprunner/hrp/internal/har2case" ) // har2caseCmd represents the har2case command diff --git a/hrp/cmd/root.go b/hrp/cmd/root.go index 2f4d866d..db06388e 100644 --- a/hrp/cmd/root.go +++ b/hrp/cmd/root.go @@ -15,9 +15,19 @@ import ( var RootCmd = &cobra.Command{ Use: "hrp", Short: "One-stop solution for HTTP(S) testing.", - Long: `hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨ + Long: ` +██╗ ██╗████████╗████████╗██████╗ ██████╗ ██╗ ██╗███╗ ██╗███╗ ██╗███████╗██████╗ +██║ ██║╚══██╔══╝╚══██╔══╝██╔══██╗██╔══██╗██║ ██║████╗ ██║████╗ ██║██╔════╝██╔══██╗ +███████║ ██║ ██║ ██████╔╝██████╔╝██║ ██║██╔██╗ ██║██╔██╗ ██║█████╗ ██████╔╝ +██╔══██║ ██║ ██║ ██╔═══╝ ██╔══██╗██║ ██║██║╚██╗██║██║╚██╗██║██╔══╝ ██╔══██╗ +██║ ██║ ██║ ██║ ██║ ██║ ██║╚██████╔╝██║ ╚████║██║ ╚████║███████╗██║ ██║ +╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝╚═╝ ╚═══╝╚══════╝╚═╝ ╚═╝ + +hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, +load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨ License: Apache-2.0 +Website: https://httprunner.com Github: https://github.com/httprunner/hrp Copyright 2021 debugtalk`, PersistentPreRun: func(cmd *cobra.Command, args []string) { diff --git a/internal/boomer/ulimit.go b/internal/boomer/ulimit.go new file mode 100644 index 00000000..b83585ea --- /dev/null +++ b/internal/boomer/ulimit.go @@ -0,0 +1,32 @@ +// +build !windows + +package boomer + +import ( + "syscall" + + "github.com/rs/zerolog/log" +) + +// set resource limit +// ulimit -n 10240 +func SetUlimit(limit uint64) { + var rLimit syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) + if err != nil { + log.Error().Err(err).Msg("get ulimit failed") + return + } + log.Info().Uint64("limit", rLimit.Cur).Msg("get current ulimit") + if rLimit.Cur >= limit { + return + } + + rLimit.Cur = limit + log.Info().Uint64("limit", rLimit.Cur).Msg("set current ulimit") + err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit) + if err != nil { + log.Error().Err(err).Msg("set ulimit failed") + return + } +} diff --git a/internal/boomer/ulimit_windows.go b/internal/boomer/ulimit_windows.go new file mode 100644 index 00000000..76ca69fc --- /dev/null +++ b/internal/boomer/ulimit_windows.go @@ -0,0 +1,12 @@ +// +build windows + +package boomer + +import ( + "github.com/rs/zerolog/log" +) + +// set resource limit +func SetUlimit(limit uint64) { + log.Warn().Msg("windows does not support setting ulimit") +} diff --git a/har2case/README.md b/internal/har2case/README.md similarity index 100% rename from har2case/README.md rename to internal/har2case/README.md diff --git a/har2case/core.go b/internal/har2case/core.go similarity index 100% rename from har2case/core.go rename to internal/har2case/core.go diff --git a/har2case/core_test.go b/internal/har2case/core_test.go similarity index 96% rename from har2case/core_test.go rename to internal/har2case/core_test.go index 43118e58..0e17cfe2 100644 --- a/har2case/core_test.go +++ b/internal/har2case/core_test.go @@ -7,8 +7,8 @@ import ( ) var ( - harPath = "../examples/har/demo.har" - harPath2 = "../examples/har/postman-echo.har" + harPath = "../../examples/har/demo.har" + harPath2 = "../../examples/har/postman-echo.har" ) func TestGenJSON(t *testing.T) { diff --git a/har2case/har.go b/internal/har2case/har.go similarity index 100% rename from har2case/har.go rename to internal/har2case/har.go diff --git a/internal/version/init.go b/internal/version/init.go index a29e664d..462025a5 100644 --- a/internal/version/init.go +++ b/internal/version/init.go @@ -1,3 +1,3 @@ package version -const VERSION = "v0.3.0" +const VERSION = "v0.4.0" diff --git a/models.go b/models.go index cbaf4b4e..34924faf 100644 --- a/models.go +++ b/models.go @@ -69,16 +69,16 @@ const ( stepTypeRendezvous stepType = "rendezvous" ) -type TransactionType string +type transactionType string const ( - TransactionStart TransactionType = "start" - TransactionEnd TransactionType = "end" + transactionStart transactionType = "start" + transactionEnd transactionType = "end" ) type Transaction struct { Name string `json:"name" yaml:"name"` - Type TransactionType `json:"type" yaml:"type"` + Type transactionType `json:"type" yaml:"type"` } type Rendezvous struct { Name string `json:"name" yaml:"name"` // required diff --git a/runner.go b/runner.go index 68782c91..a1b9d108 100644 --- a/runner.go +++ b/runner.go @@ -42,30 +42,14 @@ func NewRunner(t *testing.T) *hrpRunner { }, Timeout: 30 * time.Second, }, - sessionVariables: make(map[string]interface{}), - transactions: make(map[string]map[TransactionType]time.Time), } } type hrpRunner struct { - t *testing.T - failfast bool - debug bool - client *http.Client - sessionVariables map[string]interface{} - // transactions stores transaction timing info. - // key is transaction name, value is map of transaction type and time, e.g. start time and end time. - transactions map[string]map[TransactionType]time.Time - startTime time.Time // record start time of the testcase -} - -// Reset clears runner session variables. -func (r *hrpRunner) Reset() *hrpRunner { - log.Info().Msg("[init] Reset session variables") - r.sessionVariables = make(map[string]interface{}) - r.transactions = make(map[string]map[TransactionType]time.Time) - r.startTime = time.Now() - return r + t *testing.T + failfast bool + debug bool + client *http.Client } // SetFailfast configures whether to stop running when one step fails. @@ -108,14 +92,13 @@ func (r *hrpRunner) Run(testcases ...ITestCase) error { // report execution timing event defer ga.SendEvent(event.StartTiming("execution")) - r.Reset() for _, iTestCase := range testcases { testcase, err := iTestCase.ToTestCase() if err != nil { log.Error().Err(err).Msg("[Run] convert ITestCase interface to TestCase struct failed") return err } - if err := r.runCase(testcase); err != nil { + if err := r.newCaseRunner(testcase).run(); err != nil { log.Error().Err(err).Msg("[Run] run testcase failed") return err } @@ -123,20 +106,49 @@ func (r *hrpRunner) Run(testcases ...ITestCase) error { return nil } -func (r *hrpRunner) runCase(testcase *TestCase) error { - config := testcase.Config +func (r *hrpRunner) newCaseRunner(testcase *TestCase) *caseRunner { + caseRunner := &caseRunner{ + TestCase: testcase, + hrpRunner: r, + } + caseRunner.reset() + return caseRunner +} + +// caseRunner is used to run testcase and its steps. +// each testcase has its own caseRunner instance and share session variables. +type caseRunner struct { + *TestCase + hrpRunner *hrpRunner + sessionVariables map[string]interface{} + // transactions stores transaction timing info. + // key is transaction name, value is map of transaction type and time, e.g. start time and end time. + transactions map[string]map[transactionType]time.Time + startTime time.Time // record start time of the testcase +} + +// reset clears runner session variables. +func (r *caseRunner) reset() *caseRunner { + log.Info().Msg("[init] Reset session variables") + r.sessionVariables = make(map[string]interface{}) + r.transactions = make(map[string]map[transactionType]time.Time) + r.startTime = time.Now() + return r +} + +func (r *caseRunner) run() error { + config := r.TestCase.Config if err := r.parseConfig(config); err != nil { return err } log.Info().Str("testcase", config.Name()).Msg("run testcase start") r.startTime = time.Now() - for _, step := range testcase.TestSteps { - _, err := r.runStep(step, config) + for index := range r.TestCase.TestSteps { + _, err := r.runStep(index) if err != nil { - if r.failfast { - log.Error().Err(err).Msg("abort running due to failfast setting") - return err + if r.hrpRunner.failfast { + return errors.Wrap(err, "abort running due to failfast setting") } log.Warn().Err(err).Msg("run step failed, continue next step") } @@ -146,7 +158,10 @@ func (r *hrpRunner) runCase(testcase *TestCase) error { return nil } -func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, err error) { +func (r *caseRunner) runStep(index int) (stepResult *stepData, err error) { + config := r.TestCase.Config + step := r.TestCase.TestSteps[index] + // step type priority order: transaction > rendezvous > testcase > request if stepTran, ok := step.(*StepTransaction); ok { // transaction step @@ -218,7 +233,7 @@ func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, e return stepResult, nil } -func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *stepData, err error) { +func (r *caseRunner) runStepTransaction(transaction *Transaction) (stepResult *stepData, err error) { log.Info(). Str("name", transaction.Name). Str("type", string(transaction.Type)). @@ -234,25 +249,25 @@ func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *st // create transaction if not exists if _, ok := r.transactions[transaction.Name]; !ok { - r.transactions[transaction.Name] = make(map[TransactionType]time.Time) + r.transactions[transaction.Name] = make(map[transactionType]time.Time) } // record transaction start time, override if already exists - if transaction.Type == TransactionStart { - r.transactions[transaction.Name][TransactionStart] = time.Now() + if transaction.Type == transactionStart { + r.transactions[transaction.Name][transactionStart] = time.Now() } // record transaction end time, override if already exists - if transaction.Type == TransactionEnd { - r.transactions[transaction.Name][TransactionEnd] = time.Now() + if transaction.Type == transactionEnd { + r.transactions[transaction.Name][transactionEnd] = time.Now() // if transaction start time not exists, use testcase start time instead - if _, ok := r.transactions[transaction.Name][TransactionStart]; !ok { - r.transactions[transaction.Name][TransactionStart] = r.startTime + if _, ok := r.transactions[transaction.Name][transactionStart]; !ok { + r.transactions[transaction.Name][transactionStart] = r.startTime } // calculate transaction duration - duration := r.transactions[transaction.Name][TransactionEnd].Sub( - r.transactions[transaction.Name][TransactionStart]) + duration := r.transactions[transaction.Name][transactionEnd].Sub( + r.transactions[transaction.Name][transactionStart]) stepResult.elapsed = duration.Milliseconds() log.Info().Str("name", transaction.Name).Dur("elapsed", duration).Msg("transaction") } @@ -260,7 +275,7 @@ func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *st return stepResult, nil } -func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, err error) { +func (r *caseRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, err error) { log.Info(). Str("name", rend.Name). Float32("percent", rend.Percent). @@ -275,7 +290,7 @@ func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, e return stepResult, nil } -func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) { +func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) { stepResult = &stepData{ name: step.Name, stepType: stepTypeRequest, @@ -388,7 +403,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error req.Host = u.Host // log & print request - if r.debug { + if r.hrpRunner.debug { reqDump, err := httputil.DumpRequest(req, true) if err != nil { return nil, errors.Wrap(err, "dump request failed") @@ -399,7 +414,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error // do request action start := time.Now() - resp, err := r.client.Do(req) + resp, err := r.hrpRunner.client.Do(req) stepResult.elapsed = time.Since(start).Milliseconds() if err != nil { return nil, errors.Wrap(err, "do request failed") @@ -407,7 +422,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error defer resp.Body.Close() // log & print response - if r.debug { + if r.hrpRunner.debug { fmt.Println("==================== response ===================") respDump, err := httputil.DumpResponse(resp, true) if err != nil { @@ -418,7 +433,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error } // new response object - respObj, err := newResponseObject(r.t, resp) + respObj, err := newResponseObject(r.hrpRunner.t, resp) if err != nil { err = errors.Wrap(err, "init ResponseObject error") return @@ -443,7 +458,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error return stepResult, nil } -func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err error) { +func (r *caseRunner) runStepTestCase(step *TStep) (stepResult *stepData, err error) { stepResult = &stepData{ name: step.Name, stepType: stepTypeTestCase, @@ -451,7 +466,7 @@ func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err erro } testcase := step.TestCase start := time.Now() - err = r.runCase(testcase) + err = r.hrpRunner.newCaseRunner(testcase).run() stepResult.elapsed = time.Since(start).Milliseconds() if err != nil { return stepResult, err @@ -460,7 +475,7 @@ func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err erro return stepResult, nil } -func (r *hrpRunner) parseConfig(config IConfig) error { +func (r *caseRunner) parseConfig(config IConfig) error { cfg := config.ToStruct() // parse config variables parsedVariables, err := parseVariables(cfg.Variables) @@ -487,7 +502,7 @@ func (r *hrpRunner) parseConfig(config IConfig) error { return nil } -func (r *hrpRunner) getSummary() *testCaseSummary { +func (r *caseRunner) getSummary() *testCaseSummary { return &testCaseSummary{} } diff --git a/step.go b/step.go index 4bfcc167..dc7b3bc3 100644 --- a/step.go +++ b/step.go @@ -178,7 +178,7 @@ func (s *StepRequest) CallRefCase(tc *TestCase) *StepTestCaseWithOptionalArgs { func (s *StepRequest) StartTransaction(name string) *StepTransaction { s.step.Transaction = &Transaction{ Name: name, - Type: TransactionStart, + Type: transactionStart, } return &StepTransaction{ step: s.step, @@ -189,7 +189,7 @@ func (s *StepRequest) StartTransaction(name string) *StepTransaction { func (s *StepRequest) EndTransaction(name string) *StepTransaction { s.step.Transaction = &Transaction{ Name: name, - Type: TransactionEnd, + Type: transactionEnd, } return &StepTransaction{ step: s.step, diff --git a/step_test.go b/step_test.go index 38a91abe..5fdd3c02 100644 --- a/step_test.go +++ b/step_test.go @@ -74,12 +74,15 @@ func TestRunRequestPostDataToStruct(t *testing.T) { } func TestRunRequestRun(t *testing.T) { - config := NewConfig("test").SetBaseURL("https://postman-echo.com") - runner := NewRunner(t).SetDebug(true) - if _, err := runner.runStep(stepGET, config); err != nil { + testcase := &TestCase{ + Config: NewConfig("test").SetBaseURL("https://postman-echo.com"), + TestSteps: []IStep{stepGET, stepPOSTData}, + } + runner := NewRunner(t).SetDebug(true).newCaseRunner(testcase) + if _, err := runner.runStep(0); err != nil { t.Fatalf("tStep.Run() error: %s", err) } - if _, err := runner.runStep(stepPOSTData, config); err != nil { + if _, err := runner.runStep(1); err != nil { t.Fatalf("tStepPOSTData.Run() error: %s", err) } }