diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 3342cb3e..2ba85b0b 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -28,7 +28,7 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run coverage - run: go test -race -coverprofile="cover.out" -covermode=atomic ./... + run: go test -coverprofile="cover.out" -covermode=atomic ./... # FIXME: -race - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 with: diff --git a/README.md b/README.md index cebfab04..97650ffe 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ - [x] Supports `variables`/`extract`/`validate`/`hooks` mechanisms to create extremely complex test scenarios. - [ ] Built-in integration of rich functions, and you can also use [`go plugin`][plugin] to create and call custom functions. - [x] Inherit all powerful features of [`Boomer`][Boomer] and [`locust`][locust], you can run `load test` without extra work. -- [x] Use it as a `CLI tool` or as a `library` are both supported. +- [x] Using it as a `CLI tool` or a `library` are both supported. See [CHANGELOG]. @@ -34,7 +34,7 @@ Since installed, you will get a `hrp` command with multiple sub-commands. ```text $ hrp -h -hrp (HttpRunner+) is the one-stop solution for HTTP(S) testing. Enjoy! ✨ 🚀 ✨ +hrp (HttpRunner+) is one-stop solution for HTTP(S) testing. Enjoy! ✨ 🚀 ✨ License: Apache-2.0 Github: https://github.com/httprunner/hrp @@ -185,6 +185,7 @@ func TestCaseDemo(t *testing.T) { "varFoo2": "${max($a, $b)}", // 12.3; eval with built-in function }), TestSteps: []hrp.IStep{ + hrp.NewStep("transaction 1 start").StartTransaction("tran1"), // start transaction hrp.NewStep("get with params"). WithVariables(map[string]interface{}{ // step level variables "n": 3, // inherit config level variables if not set in step level, a/varFoo1 @@ -202,6 +203,7 @@ func TestCaseDemo(t *testing.T) { AssertLengthEqual("body.args.foo1", 5, "check args foo1"). // validate response body with jmespath AssertLengthEqual("$varFoo1", 5, "check args foo1"). // assert with extracted variable from current step AssertEqual("body.args.foo2", "34.5", "check args foo2"), // notice: request params value will be converted to string + hrp.NewStep("transaction 1 end").EndTransaction("tran1"), // end transaction hrp.NewStep("post json data"). POST("/post"). WithBody(map[string]interface{}{ diff --git a/boomer.go b/boomer.go index 4b7da8a3..fb8b5d71 100644 --- a/boomer.go +++ b/boomer.go @@ -3,12 +3,13 @@ package hrp import ( "time" - "github.com/myzhan/boomer" + "github.com/rs/zerolog/log" + "github.com/httprunner/hrp/internal/boomer" "github.com/httprunner/hrp/internal/ga" ) -func NewStandaloneBoomer(spawnCount int, spawnRate float64) *hrpBoomer { +func NewBoomer(spawnCount int, spawnRate float64) *hrpBoomer { b := &hrpBoomer{ Boomer: boomer.NewStandaloneBoomer(spawnCount, spawnRate), debug: false, @@ -50,29 +51,69 @@ func (b *hrpBoomer) Run(testcases ...ITestCase) { b.Boomer.Run(taskSlice...) } -// Quit stops running load test. -func (b *hrpBoomer) Quit() { - b.Boomer.Quit() -} - func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task { + config := testcase.Config.ToStruct() return &boomer.Task{ - Name: testcase.Config.Name, - Weight: testcase.Config.Weight, + Name: config.Name, + Weight: config.Weight, Fn: func() { - runner := NewRunner(nil).SetDebug(b.debug) - config := testcase.Config + runner := NewRunner(nil).SetDebug(b.debug).Reset() + + testcaseSuccess := true // flag whole testcase result + var transactionSuccess = true // flag current transaction result + + startTime := time.Now() for _, step := range testcase.TestSteps { - var err error - start := time.Now() - stepData, err := runner.runStep(step, config) - elapsed := time.Since(start).Nanoseconds() / int64(time.Millisecond) - if err == nil { - b.RecordSuccess(step.Type(), step.Name(), elapsed, stepData.responseLength) - } else { + stepData, err := runner.runStep(step, testcase.Config) + if err != nil { + // step failed + var elapsed int64 + if stepData != nil { + elapsed = stepData.elapsed + } b.RecordFailure(step.Type(), step.Name(), elapsed, err.Error()) + + // update flag + testcaseSuccess = false + transactionSuccess = false + + if runner.failfast { + log.Error().Err(err).Msg("abort running due to failfast setting") + break + } + log.Warn().Err(err).Msg("run step failed, continue next step") + continue + } + + // step success + if stepData.stepType == stepTypeTransaction { + // transaction + // FIXME: support nested transactions + if stepData.elapsed != 0 { // only record when transaction ends + b.RecordTransaction(stepData.name, transactionSuccess, stepData.elapsed, 0) + transactionSuccess = true // reset flag for next transaction + } + } else if stepData.stepType == stepTypeRendezvous { + // rendezvous + // TODO: implement rendezvous in boomer + } else { + // request or testcase step + b.RecordSuccess(step.Type(), step.Name(), stepData.elapsed, stepData.contentSize) } } + endTime := time.Now() + + // report duration for transaction without end + for name, transaction := range runner.transactions { + if len(transaction) == 1 { + // if transaction end time not exists, use testcase end time instead + duration := endTime.Sub(transaction[TransactionStart]) + b.RecordTransaction(name, transactionSuccess, duration.Milliseconds(), 0) + } + } + + // report testcase as a whole Action transaction, inspired by LoadRunner + b.RecordTransaction("Action", testcaseSuccess, endTime.Sub(startTime).Milliseconds(), 0) }, } } diff --git a/boomer_test.go b/boomer_test.go index 511ebfed..b49e1069 100644 --- a/boomer_test.go +++ b/boomer_test.go @@ -24,7 +24,7 @@ func TestBoomerStandaloneRun(t *testing.T) { } testcase2 := &TestCasePath{demoTestCaseJSONPath} - b := NewStandaloneBoomer(2, 1) + b := NewBoomer(2, 1) go b.Run(testcase1, testcase2) time.Sleep(5 * time.Second) b.Quit() diff --git a/convert.go b/convert.go index 978818c7..7f1bc830 100644 --- a/convert.go +++ b/convert.go @@ -7,19 +7,10 @@ import ( "io/ioutil" "path/filepath" + "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" ) -func (tc *TestCase) ToTCase() (*TCase, error) { - tCase := TCase{ - Config: tc.Config, - } - for _, step := range tc.TestSteps { - tCase.TestSteps = append(tCase.TestSteps, step.ToStruct()) - } - return &tCase, nil -} - func (tc *TCase) Dump2JSON(path string) error { path, err := filepath.Abs(path) if err != nil { @@ -105,15 +96,23 @@ func loadFromYAML(path string) (*TCase, error) { func (tc *TCase) ToTestCase() (*TestCase, error) { testCase := &TestCase{ - Config: tc.Config, + Config: &Config{cfg: tc.Config}, } for _, step := range tc.TestSteps { if step.Request != nil { - testCase.TestSteps = append(testCase.TestSteps, &requestWithOptionalArgs{ + testCase.TestSteps = append(testCase.TestSteps, &StepRequestWithOptionalArgs{ step: step, }) } else if step.TestCase != nil { - testCase.TestSteps = append(testCase.TestSteps, &testcaseWithOptionalArgs{ + testCase.TestSteps = append(testCase.TestSteps, &StepTestCaseWithOptionalArgs{ + step: step, + }) + } else if step.Transaction != nil { + testCase.TestSteps = append(testCase.TestSteps, &StepTransaction{ + step: step, + }) + } else if step.Rendezvous != nil { + testCase.TestSteps = append(testCase.TestSteps, &StepRendezvous{ step: step, }) } else { @@ -125,6 +124,11 @@ func (tc *TCase) ToTestCase() (*TestCase, error) { var ErrUnsupportedFileExt = fmt.Errorf("unsupported testcase file extension") +// TestCasePath implements ITestCase interface. +type TestCasePath struct { + Path string +} + func (path *TestCasePath) ToTestCase() (*TestCase, error) { var tc *TCase var err error diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 77bf6cd5..19770c9f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,14 @@ # Release History +## v0.3.0 (2021-12-22) + +- feat: implement `transaction` mechanism for load test +- feat: support `--continue-on-failure` flag to continue running next step when failure occurs, default to failfast +- refactor: fork [boomer] as sub module +- feat: report GA events with version +- feat: run load test with the given limit and burst as rate limiter +- change: update API models + ## v0.2.2 (2021-12-07) - refactor: update models to make API more concise diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index 21663901..a8884d6b 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -4,7 +4,7 @@ One-stop solution for HTTP(S) testing. ### Synopsis -hrp (HttpRunner+) is the next generation for HttpRunner. Enjoy! ✨ 🚀 ✨ +hrp (HttpRunner+) is one-stop solution for HTTP(S) testing. Enjoy! ✨ 🚀 ✨ License: Apache-2.0 Github: https://github.com/httprunner/hrp @@ -22,4 +22,4 @@ Copyright 2021 debugtalk * [hrp har2case](hrp_har2case.md) - Convert HAR to json/yaml testcase files * [hrp run](hrp_run.md) - run API test -###### Auto generated by spf13/cobra on 7-Dec-2021 +###### Auto generated by spf13/cobra on 23-Dec-2021 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index 8c906a3a..f61e625c 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -30,7 +30,6 @@ hrp boom [flags] --mem-profile-duration duration Memory profile duration. (default 30s) --prometheus-gateway string Prometheus Pushgateway url. --request-increase-rate string Request increase rate, disabled by default. (default "-1") - --run-tasks string Run tasks without connecting to the master, multiply tasks is separated by comma. Usually, it's for debug purpose. --spawn-count int The number of users to spawn for load testing (default 1) --spawn-rate float The rate for spawning users (default 1) ``` @@ -39,4 +38,4 @@ hrp boom [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 7-Dec-2021 +###### Auto generated by spf13/cobra on 23-Dec-2021 diff --git a/docs/cmd/hrp_har2case.md b/docs/cmd/hrp_har2case.md index 0932d403..30afecc9 100644 --- a/docs/cmd/hrp_har2case.md +++ b/docs/cmd/hrp_har2case.md @@ -23,4 +23,4 @@ hrp har2case harPath... [flags] * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 7-Dec-2021 +###### Auto generated by spf13/cobra on 23-Dec-2021 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index 18ba8a02..26f75e9d 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -21,13 +21,14 @@ hrp run path... [flags] ### Options ``` - -h, --help help for run - -p, --proxy-url string set proxy url - -s, --silent disable logging request & response details + --continue-on-failure continue running next step when failure occurs + -h, --help help for run + -p, --proxy-url string set proxy url + -s, --silent disable logging request & response details ``` ### SEE ALSO * [hrp](hrp.md) - One-stop solution for HTTP(S) testing. -###### Auto generated by spf13/cobra on 7-Dec-2021 +###### Auto generated by spf13/cobra on 23-Dec-2021 diff --git a/examples/demo.json b/examples/demo.json index 4f371a05..0f6c9819 100644 --- a/examples/demo.json +++ b/examples/demo.json @@ -11,6 +11,13 @@ } }, "teststeps": [ + { + "name": "transaction 1 start", + "transaction": { + "name": "tran1", + "type": "start" + } + }, { "name": "get with params", "request": { @@ -64,6 +71,13 @@ } ] }, + { + "name": "transaction 1 end", + "transaction": { + "name": "tran1", + "type": "end" + } + }, { "name": "post json data", "request": { diff --git a/examples/demo.yaml b/examples/demo.yaml index 0bc18920..a0cee432 100644 --- a/examples/demo.yaml +++ b/examples/demo.yaml @@ -8,6 +8,10 @@ config: varFoo1: ${gen_random_string($n)} varFoo2: ${max($a, $b)} teststeps: + - name: transaction 1 start + transaction: + name: tran1 + type: start - name: get with params request: method: GET @@ -43,6 +47,10 @@ teststeps: assert: equals expect: "34.5" msg: check args foo2 + - name: transaction 1 end + transaction: + name: tran1 + type: end - name: post json data request: method: POST diff --git a/examples/demo_test.go b/examples/demo_test.go index 0fb4e566..bb41544e 100644 --- a/examples/demo_test.go +++ b/examples/demo_test.go @@ -18,6 +18,7 @@ var demoTestCase = &hrp.TestCase{ "varFoo2": "${max($a, $b)}", // 12.3; eval with built-in function }), TestSteps: []hrp.IStep{ + hrp.NewStep("transaction 1 start").StartTransaction("tran1"), // start transaction hrp.NewStep("get with params"). WithVariables(map[string]interface{}{ // step level variables "n": 3, // inherit config level variables if not set in step level, a/varFoo1 @@ -35,6 +36,7 @@ var demoTestCase = &hrp.TestCase{ AssertLengthEqual("body.args.foo1", 5, "check args foo1"). // validate response body with jmespath AssertLengthEqual("$varFoo1", 5, "check args foo1"). // assert with extracted variable from current step AssertEqual("body.args.foo2", "34.5", "check args foo2"), // notice: request params value will be converted to string + hrp.NewStep("transaction 1 end").EndTransaction("tran1"), // end transaction hrp.NewStep("post json data"). POST("/post"). WithBody(map[string]interface{}{ diff --git a/examples/demo_test.py b/examples/demo_test.py new file mode 100644 index 00000000..e2eddc1f --- /dev/null +++ b/examples/demo_test.py @@ -0,0 +1,63 @@ +# NOTE: Generated By HttpRunner v3.1.6 +# FROM: hrp/examples/demo.json + + +from httprunner import HttpRunner, Config, Step, RunRequest, RunTestCase + + +class TestCaseDemo(HttpRunner): + + config = ( + Config("demo with complex mechanisms") + .variables( + **{ + "a": 12.3, + "b": 3.45, + "n": 5, + "varFoo1": "${gen_random_string($n)}", + "varFoo2": "${max($a, $b)}", + } + ) + .base_url("https://postman-echo.com") + ) + + teststeps = [ + Step( + RunRequest("get with params") + .with_variables(**{"b": 34.5, "n": 3, "varFoo2": "${max($a, $b)}"}) + .get("/get") + .with_params(**{"foo1": "$varFoo1", "foo2": "$varFoo2"}) + .with_headers(**{"User-Agent": "HttpRunnerPlus"}) + .extract() + .with_jmespath("body.args.foo1", "varFoo1") + .validate() + .assert_equal("status_code", 200) + .assert_equal('headers."Content-Type"', "application/json") + .assert_equal("body.args.foo1", 5) + .assert_equal("$varFoo1", 5) + .assert_equal("body.args.foo2", "34.5") + ), + Step( + RunRequest("post json data") + .post("/post") + .validate() + .assert_equal("status_code", 200) + .assert_equal("body.json.foo1", 5) + .assert_equal("body.json.foo2", 12.3) + ), + Step( + RunRequest("post form data") + .post("/post") + .with_headers( + **{"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"} + ) + .validate() + .assert_equal("status_code", 200) + .assert_equal("body.form.foo1", 5) + .assert_equal("body.form.foo2", "12.3") + ), + ] + + +if __name__ == "__main__": + TestCaseDemo().test_start() diff --git a/extract.go b/extract.go index 0d0fb388..b4269939 100644 --- a/extract.go +++ b/extract.go @@ -2,32 +2,32 @@ package hrp import "fmt" -// implements IStep interface -type stepRequestExtraction struct { +// StepRequestExtraction implements IStep interface. +type StepRequestExtraction struct { step *TStep } // WithJmesPath sets the JMESPath expression to extract from the response. -func (s *stepRequestExtraction) WithJmesPath(jmesPath string, varName string) *stepRequestExtraction { +func (s *StepRequestExtraction) WithJmesPath(jmesPath string, varName string) *StepRequestExtraction { s.step.Extract[varName] = jmesPath return s } // Validate switches to step validation. -func (s *stepRequestExtraction) Validate() *stepRequestValidation { - return &stepRequestValidation{ +func (s *StepRequestExtraction) Validate() *StepRequestValidation { + return &StepRequestValidation{ step: s.step, } } -func (s *stepRequestExtraction) Name() string { +func (s *StepRequestExtraction) Name() string { return s.step.Name } -func (s *stepRequestExtraction) Type() string { +func (s *StepRequestExtraction) Type() string { return fmt.Sprintf("request-%v", s.step.Request.Method) } -func (s *stepRequestExtraction) ToStruct() *TStep { +func (s *StepRequestExtraction) ToStruct() *TStep { return s.step } diff --git a/go.mod b/go.mod index 69797d2e..6bfe7528 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,14 @@ go 1.13 require ( github.com/denisbrodbeck/machineid v1.0.1 github.com/getsentry/sentry-go v0.11.0 - github.com/go-ole/go-ole v1.2.6 // indirect github.com/google/uuid v1.3.0 github.com/jinzhu/copier v0.3.2 github.com/jmespath/go-jmespath v0.4.0 github.com/maja42/goval v1.2.1 github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/myzhan/boomer v1.6.1-0.20211202034203-f3ce8f55124f + github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.11.0 github.com/rs/zerolog v1.26.0 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 1b83e2f2..c5a10d67 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,6 @@ github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3 github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= -github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= -github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -57,8 +55,6 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= -github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -128,9 +124,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= -github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= @@ -313,8 +306,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/myzhan/boomer v1.6.1-0.20211202034203-f3ce8f55124f h1:y1EahE5P+fP8e05QJR5cSMJaEwUVuijzydoXAQlVH1E= -github.com/myzhan/boomer v1.6.1-0.20211202034203-f3ce8f55124f/go.mod h1:vJdhrrbJAYGcr7qDAtxanOrPj7W6qyzMMTHsENWEs1o= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= @@ -372,8 +363,6 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v3.21.10+incompatible h1:AL2kpVykjkqeN+MFe1WcwSBVUjGjvdU8/ubvCuXAjrU= -github.com/shirou/gopsutil v3.21.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -405,18 +394,10 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= -github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= -github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= -github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go v1.2.6 h1:tGiWC9HENWE2tqYycIqFTNorMmFRVhNwCpDOpWqnk8E= -github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ= -github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= @@ -436,12 +417,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/zeromq/goczmq v0.0.0-20190906225145-a7546843a315 h1:Mnki1bwiVDLVh9/gMqjI+3MdbVmAbswzayK/bzRmNaE= -github.com/zeromq/goczmq v0.0.0-20190906225145-a7546843a315/go.mod h1:jBJgSEDlcqrdShbpgYc2S+mTo1Rs6pac+8zpUQFgsvg= -github.com/zeromq/gomq v0.0.0-20201031135124-cef4e507bb8e h1:vGjfCnWv/zWeO1ivv4+OUPgTzG/WV1iGfZwVdtUpLkM= -github.com/zeromq/gomq v0.0.0-20201031135124-cef4e507bb8e/go.mod h1:SkCxcSQ7BQEA9FvDzbj+3hV6EMhSywyxWnHwUXVIyLY= -github.com/zeromq/gomq/zmtp v0.0.0-20201031135124-cef4e507bb8e h1:pjp04/sSr2TYuaPdt+u6Cc1M38Aocp+3er0akr3auFg= -github.com/zeromq/gomq/zmtp v0.0.0-20201031135124-cef4e507bb8e/go.mod h1:LBjWEodY/ESvKRwLw3bc7mhn49oiI8qlXUqeqLn0pcU= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -586,7 +561,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -624,8 +598,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/har2case/core.go b/har2case/core.go index 5271fc15..b385efc7 100644 --- a/har2case/core.go +++ b/har2case/core.go @@ -12,21 +12,18 @@ import ( "strings" "github.com/pkg/errors" - "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/httprunner/hrp" "github.com/httprunner/hrp/internal/ga" ) -var log zerolog.Logger - const ( suffixJSON = ".json" suffixYAML = ".yaml" ) func NewHAR(path string) *har { - log = hrp.GetLogger() return &har{ path: path, } @@ -117,7 +114,7 @@ func (h *har) load() (*Har, error) { func (h *har) prepareConfig() *hrp.TConfig { return hrp.NewConfig("testcase description"). - SetVerifySSL(false) + SetVerifySSL(false).ToStruct() } func (h *har) prepareTestSteps() ([]*hrp.TStep, error) { diff --git a/hrp/cmd/boom.go b/hrp/cmd/boom.go index e74d3d55..41c67758 100644 --- a/hrp/cmd/boom.go +++ b/hrp/cmd/boom.go @@ -3,10 +3,10 @@ package cmd import ( "time" - "github.com/myzhan/boomer" "github.com/spf13/cobra" "github.com/httprunner/hrp" + "github.com/httprunner/hrp/internal/boomer" ) // boomCmd represents the boom command @@ -19,14 +19,15 @@ var boomCmd = &cobra.Command{ $ hrp boom examples/ # run testcases in specified folder`, Args: cobra.MinimumNArgs(1), PreRun: func(cmd *cobra.Command, args []string) { - hrp.SetLogger("WARN", logJSON) // disable info logs for load testing + setLogLevel("WARN") // disable info logs for load testing }, Run: func(cmd *cobra.Command, args []string) { var paths []hrp.ITestCase for _, arg := range args { paths = append(paths, &hrp.TestCasePath{Path: arg}) } - hrpBoomer := hrp.NewStandaloneBoomer(spawnCount, spawnRate) + hrpBoomer := hrp.NewBoomer(spawnCount, spawnRate) + hrpBoomer.SetRateLimiter(maxRPS, requestIncreaseRate) if !disableConsoleOutput { hrpBoomer.AddOutput(boomer.NewConsoleOutput()) } @@ -42,9 +43,8 @@ var boomCmd = &cobra.Command{ var ( spawnCount int spawnRate float64 - maxRPS int64 // TODO: init boomer with this flag - requestIncreaseRate string // TODO: init boomer with this flag - runTasks string // TODO: init boomer with this flag + maxRPS int64 + requestIncreaseRate string memoryProfile string memoryProfileDuration time.Duration cpuProfile string @@ -58,7 +58,6 @@ func init() { boomCmd.Flags().Int64Var(&maxRPS, "max-rps", 0, "Max RPS that boomer can generate, disabled by default.") boomCmd.Flags().StringVar(&requestIncreaseRate, "request-increase-rate", "-1", "Request increase rate, disabled by default.") - boomCmd.Flags().StringVar(&runTasks, "run-tasks", "", "Run tasks without connecting to the master, multiply tasks is separated by comma. Usually, it's for debug purpose.") boomCmd.Flags().IntVar(&spawnCount, "spawn-count", 1, "The number of users to spawn for load testing") boomCmd.Flags().Float64Var(&spawnRate, "spawn-rate", 1, "The rate for spawning users") boomCmd.Flags().StringVar(&memoryProfile, "mem-profile", "", "Enable memory profiling.") diff --git a/hrp/cmd/har2case.go b/hrp/cmd/har2case.go index a3fc47ba..73670a57 100644 --- a/hrp/cmd/har2case.go +++ b/hrp/cmd/har2case.go @@ -1,9 +1,9 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "github.com/httprunner/hrp" "github.com/httprunner/hrp/har2case" ) @@ -13,6 +13,9 @@ var har2caseCmd = &cobra.Command{ Short: "Convert HAR to json/yaml testcase files", Long: `Convert HAR to json/yaml testcase files`, Args: cobra.MinimumNArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + setLogLevel(logLevel) + }, RunE: func(cmd *cobra.Command, args []string) error { var outputFiles []string for _, arg := range args { @@ -37,7 +40,6 @@ var har2caseCmd = &cobra.Command{ } outputFiles = append(outputFiles, outputPath) } - log := hrp.GetLogger() log.Info().Strs("output", outputFiles).Msg("convert testcase success") return nil }, diff --git a/hrp/cmd/root.go b/hrp/cmd/root.go index c49caada..3db69816 100644 --- a/hrp/cmd/root.go +++ b/hrp/cmd/root.go @@ -1,12 +1,13 @@ package cmd import ( - "fmt" "os" + "strings" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "github.com/httprunner/hrp" "github.com/httprunner/hrp/internal/version" ) @@ -14,13 +15,16 @@ import ( var RootCmd = &cobra.Command{ Use: "hrp", Short: "One-stop solution for HTTP(S) testing.", - Long: `hrp (HttpRunner+) is the one-stop solution for HTTP(S) testing. Enjoy! ✨ 🚀 ✨ + Long: `hrp (HttpRunner+) is one-stop solution for HTTP(S) testing. Enjoy! ✨ 🚀 ✨ License: Apache-2.0 Github: https://github.com/httprunner/hrp Copyright 2021 debugtalk`, PersistentPreRun: func(cmd *cobra.Command, args []string) { - hrp.SetLogger(logLevel, logJSON) + if !logJSON { + log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger() + log.Info().Msg("Set log to color console other than JSON format.") + } }, Version: version.VERSION, } @@ -37,7 +41,26 @@ func Execute() { RootCmd.PersistentFlags().BoolVar(&logJSON, "log-json", false, "set log to json format") if err := RootCmd.Execute(); err != nil { - fmt.Println(err) + log.Error().Err(err).Msg("Failed to execute root command") os.Exit(1) } } + +func setLogLevel(level string) { + level = strings.ToUpper(level) + log.Info().Str("level", level).Msg("Set log level") + switch level { + case "DEBUG": + zerolog.SetGlobalLevel(zerolog.DebugLevel) + case "INFO": + zerolog.SetGlobalLevel(zerolog.InfoLevel) + case "WARN": + zerolog.SetGlobalLevel(zerolog.WarnLevel) + case "ERROR": + zerolog.SetGlobalLevel(zerolog.ErrorLevel) + case "FATAL": + zerolog.SetGlobalLevel(zerolog.FatalLevel) + case "PANIC": + zerolog.SetGlobalLevel(zerolog.PanicLevel) + } +} diff --git a/hrp/cmd/run.go b/hrp/cmd/run.go index 6cf20870..93423094 100644 --- a/hrp/cmd/run.go +++ b/hrp/cmd/run.go @@ -15,12 +15,17 @@ var runCmd = &cobra.Command{ $ hrp run demo.yaml # run specified yaml testcase file $ hrp run examples/ # run testcases in specified folder`, Args: cobra.MinimumNArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + setLogLevel(logLevel) + }, RunE: func(cmd *cobra.Command, args []string) error { var paths []hrp.ITestCase for _, arg := range args { paths = append(paths, &hrp.TestCasePath{Path: arg}) } - runner := hrp.NewRunner(nil).SetDebug(!silentFlag) + runner := hrp.NewRunner(nil). + SetDebug(!silentFlag). + SetFailfast(!continueOnFailure) if proxyUrl != "" { runner.SetProxyUrl(proxyUrl) } @@ -29,12 +34,14 @@ var runCmd = &cobra.Command{ } var ( - silentFlag bool - proxyUrl string + continueOnFailure bool + silentFlag bool + proxyUrl string ) func init() { RootCmd.AddCommand(runCmd) + runCmd.Flags().BoolVar(&continueOnFailure, "continue-on-failure", false, "continue running next step when failure occurs") runCmd.Flags().BoolVarP(&silentFlag, "silent", "s", false, "disable logging request & response details") runCmd.Flags().StringVarP(&proxyUrl, "proxy-url", "p", "", "set proxy url") // runCmd.Flags().BoolP("gen-html-report", "r", false, "Generate HTML report") diff --git a/internal/boomer/README.md b/internal/boomer/README.md new file mode 100644 index 00000000..b6ef5ce2 --- /dev/null +++ b/internal/boomer/README.md @@ -0,0 +1,5 @@ +# boomer + +This module is initially forked from [myzhan/boomer] and made a lot of changes. + +[myzhan/boomer]: https://github.com/myzhan/boomer diff --git a/internal/boomer/boomer.go b/internal/boomer/boomer.go new file mode 100644 index 00000000..3f50a60b --- /dev/null +++ b/internal/boomer/boomer.go @@ -0,0 +1,146 @@ +package boomer + +import ( + "math" + "time" + + "github.com/rs/zerolog/log" +) + +// A Boomer is used to run tasks. +type Boomer struct { + rateLimiter RateLimiter + + localRunner *localRunner + spawnCount int + spawnRate float64 + + cpuProfile string + cpuProfileDuration time.Duration + + memoryProfile string + memoryProfileDuration time.Duration + + outputs []Output +} + +// NewStandaloneBoomer returns a new Boomer, which can run without master. +func NewStandaloneBoomer(spawnCount int, spawnRate float64) *Boomer { + return &Boomer{ + spawnCount: spawnCount, + spawnRate: spawnRate, + } +} + +// SetRateLimiter creates rate limiter with the given limit and burst. +func (b *Boomer) SetRateLimiter(maxRPS int64, requestIncreaseRate string) { + var rateLimiter RateLimiter + var err error + if requestIncreaseRate != "-1" { + if maxRPS <= 0 { + maxRPS = math.MaxInt64 + } + log.Warn().Int64("maxRPS", maxRPS).Str("increaseRate", requestIncreaseRate).Msg("set ramp up rate limiter") + rateLimiter, err = NewRampUpRateLimiter(maxRPS, requestIncreaseRate, time.Second) + } else { + if maxRPS > 0 { + log.Warn().Int64("maxRPS", maxRPS).Msg("set stable rate limiter") + rateLimiter = NewStableRateLimiter(maxRPS, time.Second) + } + } + if err != nil { + log.Error().Err(err).Msg("failed to create rate limiter") + return + } + b.rateLimiter = rateLimiter +} + +// AddOutput accepts outputs which implements the boomer.Output interface. +func (b *Boomer) AddOutput(o Output) { + b.outputs = append(b.outputs, o) +} + +// EnableCPUProfile will start cpu profiling after run. +func (b *Boomer) EnableCPUProfile(cpuProfile string, duration time.Duration) { + b.cpuProfile = cpuProfile + b.cpuProfileDuration = duration +} + +// EnableMemoryProfile will start memory profiling after run. +func (b *Boomer) EnableMemoryProfile(memoryProfile string, duration time.Duration) { + b.memoryProfile = memoryProfile + b.memoryProfileDuration = duration +} + +// Run accepts a slice of Task and connects to the locust master. +func (b *Boomer) Run(tasks ...*Task) { + if b.cpuProfile != "" { + err := startCPUProfile(b.cpuProfile, b.cpuProfileDuration) + if err != nil { + log.Error().Err(err).Msg("failed to start cpu profiling") + } + } + if b.memoryProfile != "" { + err := startMemoryProfile(b.memoryProfile, b.memoryProfileDuration) + if err != nil { + log.Error().Err(err).Msg("failed to start memory profiling") + } + } + + b.localRunner = newLocalRunner(tasks, b.rateLimiter, b.spawnCount, b.spawnRate) + for _, o := range b.outputs { + b.localRunner.addOutput(o) + } + b.localRunner.run() +} + +// RecordTransaction reports a transaction stat. +func (b *Boomer) RecordTransaction(name string, success bool, elapsedTime int64, contentSize int64) { + if b.localRunner == nil { + log.Warn().Msg("boomer not initialized") + return + } + b.localRunner.stats.transactionChan <- &transaction{ + name: name, + success: success, + elapsedTime: elapsedTime, + contentSize: contentSize, + } +} + +// RecordSuccess reports a success. +func (b *Boomer) RecordSuccess(requestType, name string, responseTime int64, responseLength int64) { + if b.localRunner == nil { + log.Warn().Msg("boomer not initialized") + return + } + b.localRunner.stats.requestSuccessChan <- &requestSuccess{ + requestType: requestType, + name: name, + responseTime: responseTime, + responseLength: responseLength, + } +} + +// RecordFailure reports a failure. +func (b *Boomer) RecordFailure(requestType, name string, responseTime int64, exception string) { + if b.localRunner == nil { + log.Warn().Msg("boomer not initialized") + return + } + b.localRunner.stats.requestFailureChan <- &requestFailure{ + requestType: requestType, + name: name, + responseTime: responseTime, + errMsg: exception, + } +} + +// Quit will send a quit message to the master. +func (b *Boomer) Quit() { + if b.localRunner == nil { + log.Warn().Msg("boomer not initialized") + return + } + b.localRunner.close() +} diff --git a/internal/boomer/boomer_test.go b/internal/boomer/boomer_test.go new file mode 100644 index 00000000..5670c763 --- /dev/null +++ b/internal/boomer/boomer_test.go @@ -0,0 +1,146 @@ +package boomer + +import ( + "math" + "os" + "runtime" + "sync/atomic" + "testing" + "time" +) + +func TestNewStandaloneBoomer(t *testing.T) { + b := NewStandaloneBoomer(100, 10) + + if b.spawnCount != 100 { + t.Error("spawnCount should be 100") + } + + if b.spawnRate != 10 { + t.Error("spawnRate should be 10") + } +} + +func TestSetRateLimiter(t *testing.T) { + b := NewStandaloneBoomer(100, 10) + b.SetRateLimiter(10, "10/1s") + + if b.rateLimiter == nil { + t.Error("b.rateLimiter should not be nil") + } +} + +func TestAddOutput(t *testing.T) { + b := NewStandaloneBoomer(100, 10) + b.AddOutput(NewConsoleOutput()) + b.AddOutput(NewConsoleOutput()) + + if len(b.outputs) != 2 { + t.Error("length of outputs should be 2") + } +} + +func TestEnableCPUProfile(t *testing.T) { + b := NewStandaloneBoomer(100, 10) + b.EnableCPUProfile("cpu.prof", time.Second) + + if b.cpuProfile != "cpu.prof" { + t.Error("cpuProfile should be cpu.prof") + } + + if b.cpuProfileDuration != time.Second { + t.Error("cpuProfileDuration should 1 second") + } +} + +func TestEnableMemoryProfile(t *testing.T) { + b := NewStandaloneBoomer(100, 10) + b.EnableMemoryProfile("mem.prof", time.Second) + + if b.memoryProfile != "mem.prof" { + t.Error("memoryProfile should be mem.prof") + } + + if b.memoryProfileDuration != time.Second { + t.Error("memoryProfileDuration should 1 second") + } +} + +func TestStandaloneRun(t *testing.T) { + b := NewStandaloneBoomer(10, 10) + b.EnableCPUProfile("cpu.pprof", 2*time.Second) + b.EnableMemoryProfile("mem.pprof", 2*time.Second) + + count := int64(0) + taskA := &Task{ + Name: "increaseCount", + Fn: func() { + atomic.AddInt64(&count, 1) + runtime.Goexit() + }, + } + go b.Run(taskA) + + time.Sleep(5 * time.Second) + + b.Quit() + + if count != 10 { + t.Error("count is", count, "expected: 10") + } + + if _, err := os.Stat("cpu.pprof"); os.IsNotExist(err) { + t.Error("File cpu.pprof is not generated") + } else { + os.Remove("cpu.pprof") + } + + if _, err := os.Stat("mem.pprof"); os.IsNotExist(err) { + t.Error("File mem.pprof is not generated") + } else { + os.Remove("mem.pprof") + } +} + +func TestCreateRatelimiter(t *testing.T) { + b := NewStandaloneBoomer(10, 10) + b.SetRateLimiter(100, "-1") + + if stableRateLimiter, ok := b.rateLimiter.(*StableRateLimiter); !ok { + t.Error("Expected stableRateLimiter") + } else { + if stableRateLimiter.threshold != 100 { + t.Error("threshold should be equals to math.MaxInt64, was", stableRateLimiter.threshold) + } + } + + b.SetRateLimiter(0, "1") + if rampUpRateLimiter, ok := b.rateLimiter.(*RampUpRateLimiter); !ok { + t.Error("Expected rampUpRateLimiter") + } else { + if rampUpRateLimiter.maxThreshold != math.MaxInt64 { + t.Error("maxThreshold should be equals to math.MaxInt64, was", rampUpRateLimiter.maxThreshold) + } + if rampUpRateLimiter.rampUpRate != "1" { + t.Error("rampUpRate should be equals to \"1\", was", rampUpRateLimiter.rampUpRate) + } + } + + b.SetRateLimiter(10, "2/2s") + if rampUpRateLimiter, ok := b.rateLimiter.(*RampUpRateLimiter); !ok { + t.Error("Expected rampUpRateLimiter") + } else { + if rampUpRateLimiter.maxThreshold != 10 { + t.Error("maxThreshold should be equals to 10, was", rampUpRateLimiter.maxThreshold) + } + if rampUpRateLimiter.rampUpRate != "2/2s" { + t.Error("rampUpRate should be equals to \"2/2s\", was", rampUpRateLimiter.rampUpRate) + } + if rampUpRateLimiter.rampUpStep != 2 { + t.Error("rampUpStep should be equals to 2, was", rampUpRateLimiter.rampUpStep) + } + if rampUpRateLimiter.rampUpPeroid != 2*time.Second { + t.Error("rampUpPeroid should be equals to 2 seconds, was", rampUpRateLimiter.rampUpPeroid) + } + } +} diff --git a/internal/boomer/output.go b/internal/boomer/output.go new file mode 100644 index 00000000..3f897e91 --- /dev/null +++ b/internal/boomer/output.go @@ -0,0 +1,431 @@ +package boomer + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strconv" + "time" + + "github.com/google/uuid" + "github.com/olekukonko/tablewriter" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/push" + "github.com/rs/zerolog/log" +) + +// Output is primarily responsible for printing test results to different destinations +// such as consoles, files. You can write you own output and add to boomer. +// When running in standalone mode, the default output is ConsoleOutput, you can add more. +// When running in distribute mode, test results will be reported to master with or without +// an output. +// All the OnXXX function will be call in a separated goroutine, just in case some output will block. +// But it will wait for all outputs return to avoid data lost. +type Output interface { + // OnStart will be call before the test starts. + OnStart() + + // By default, each output receive stats data from runner every three seconds. + // OnEvent is responsible for dealing with the data. + OnEvent(data map[string]interface{}) + + // OnStop will be called before the test ends. + OnStop() +} + +// ConsoleOutput is the default output for standalone mode. +type ConsoleOutput struct { +} + +// NewConsoleOutput returns a ConsoleOutput. +func NewConsoleOutput() *ConsoleOutput { + return &ConsoleOutput{} +} + +func getMedianResponseTime(numRequests int64, responseTimes map[int64]int64) int64 { + medianResponseTime := int64(0) + if len(responseTimes) != 0 { + pos := (numRequests - 1) / 2 + var sortedKeys []int64 + for k := range responseTimes { + sortedKeys = append(sortedKeys, k) + } + sort.Slice(sortedKeys, func(i, j int) bool { + return sortedKeys[i] < sortedKeys[j] + }) + for _, k := range sortedKeys { + if pos < responseTimes[k] { + medianResponseTime = k + break + } + pos -= responseTimes[k] + } + } + return medianResponseTime +} + +func getAvgResponseTime(numRequests int64, totalResponseTime int64) (avgResponseTime float64) { + avgResponseTime = float64(0) + if numRequests != 0 { + avgResponseTime = float64(totalResponseTime) / float64(numRequests) + } + return avgResponseTime +} + +func getAvgContentLength(numRequests int64, totalContentLength int64) (avgContentLength int64) { + avgContentLength = int64(0) + if numRequests != 0 { + avgContentLength = totalContentLength / numRequests + } + return avgContentLength +} + +func getCurrentRps(numRequests int64, numReqsPerSecond map[int64]int64) (currentRps int64) { + currentRps = int64(0) + numReqsPerSecondLength := int64(len(numReqsPerSecond)) + if numReqsPerSecondLength != 0 { + currentRps = numRequests / numReqsPerSecondLength + } + return currentRps +} + +func getCurrentFailPerSec(numFailures int64, numFailPerSecond map[int64]int64) (currentFailPerSec int64) { + currentFailPerSec = int64(0) + numFailPerSecondLength := int64(len(numFailPerSecond)) + if numFailPerSecondLength != 0 { + currentFailPerSec = numFailures / numFailPerSecondLength + } + return currentFailPerSec +} + +func getTotalFailRatio(totalRequests, totalFailures int64) (failRatio float64) { + if totalRequests == 0 { + return 0 + } + return float64(totalFailures) / float64(totalRequests) +} + +// OnStart of ConsoleOutput has nothing to do. +func (o *ConsoleOutput) OnStart() { + +} + +// OnStop of ConsoleOutput has nothing to do. +func (o *ConsoleOutput) OnStop() { + +} + +// OnEvent will print to the console. +func (o *ConsoleOutput) OnEvent(data map[string]interface{}) { + output, err := convertData(data) + if err != nil { + log.Error().Err(err).Msg("failed to convert data") + return + } + + currentTime := time.Now() + println(fmt.Sprintf("Current time: %s, Users: %d, Total RPS: %d, Total Fail Ratio: %.1f%%", + currentTime.Format("2006/01/02 15:04:05"), output.UserCount, output.TotalRPS, output.TotalFailRatio*100)) + println(fmt.Sprintf("Accumulated Transactions: %d Passed, %d Failed", + output.TransactionsPassed, output.TransactionsFailed)) + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"Type", "Name", "# requests", "# fails", "Median", "Average", "Min", "Max", "Content Size", "# reqs/sec", "# fails/sec"}) + + for _, stat := range output.Stats { + row := make([]string, 11) + row[0] = stat.Method + row[1] = stat.Name + row[2] = strconv.FormatInt(stat.NumRequests, 10) + row[3] = strconv.FormatInt(stat.NumFailures, 10) + row[4] = strconv.FormatInt(stat.medianResponseTime, 10) + row[5] = strconv.FormatFloat(stat.avgResponseTime, 'f', 2, 64) + row[6] = strconv.FormatInt(stat.MinResponseTime, 10) + row[7] = strconv.FormatInt(stat.MaxResponseTime, 10) + row[8] = strconv.FormatInt(stat.avgContentLength, 10) + row[9] = strconv.FormatInt(stat.currentRps, 10) + row[10] = strconv.FormatInt(stat.currentFailPerSec, 10) + table.Append(row) + } + table.Render() + println() +} + +type statsEntryOutput struct { + statsEntry + + medianResponseTime int64 // median response time + avgResponseTime float64 // average response time, round float to 2 decimal places + avgContentLength int64 // average content size + currentRps int64 // # reqs/sec + currentFailPerSec int64 // # fails/sec +} + +type dataOutput struct { + UserCount int32 `json:"user_count"` + TotalStats *statsEntryOutput `json:"stats_total"` + TransactionsPassed int64 `json:"transactions_passed"` + TransactionsFailed int64 `json:"transactions_failed"` + TotalRPS int64 `json:"total_rps"` + TotalFailRatio float64 `json:"total_fail_ratio"` + Stats []*statsEntryOutput `json:"stats"` + Errors map[string]map[string]interface{} `json:"errors"` +} + +func convertData(data map[string]interface{}) (output *dataOutput, err error) { + userCount, ok := data["user_count"].(int32) + if !ok { + return nil, fmt.Errorf("user_count is not int32") + } + stats, ok := data["stats"].([]interface{}) + if !ok { + return nil, fmt.Errorf("stats is not []interface{}") + } + + transactions, ok := data["transactions"].(map[string]int64) + if !ok { + return nil, fmt.Errorf("transactions is not map[string]int64") + } + transactionsPassed := transactions["passed"] + transactionsFailed := transactions["failed"] + + // convert stats in total + statsTotal, ok := data["stats_total"].(interface{}) + if !ok { + return nil, fmt.Errorf("stats_total is not interface{}") + } + entryTotalOutput, err := deserializeStatsEntry(statsTotal) + if err != nil { + return nil, err + } + + output = &dataOutput{ + UserCount: userCount, + TotalStats: entryTotalOutput, + TransactionsPassed: transactionsPassed, + TransactionsFailed: transactionsFailed, + TotalRPS: getCurrentRps(entryTotalOutput.NumRequests, entryTotalOutput.NumReqsPerSec), + TotalFailRatio: getTotalFailRatio(entryTotalOutput.NumRequests, entryTotalOutput.NumFailures), + Stats: make([]*statsEntryOutput, 0, len(stats)), + } + + // convert stats + for _, stat := range stats { + entryOutput, err := deserializeStatsEntry(stat) + if err != nil { + return nil, err + } + output.Stats = append(output.Stats, entryOutput) + } + // sort stats by type + sort.Slice(output.Stats, func(i, j int) bool { + return output.Stats[i].Method < output.Stats[j].Method + }) + return +} + +func deserializeStatsEntry(stat interface{}) (entryOutput *statsEntryOutput, err error) { + statBytes, err := json.Marshal(stat) + if err != nil { + return nil, err + } + entry := statsEntry{} + if err = json.Unmarshal(statBytes, &entry); err != nil { + return nil, err + } + + numRequests := entry.NumRequests + entryOutput = &statsEntryOutput{ + statsEntry: entry, + medianResponseTime: getMedianResponseTime(numRequests, entry.ResponseTimes), + avgResponseTime: getAvgResponseTime(numRequests, entry.TotalResponseTime), + avgContentLength: getAvgContentLength(numRequests, entry.TotalContentLength), + currentRps: getCurrentRps(numRequests, entry.NumReqsPerSec), + currentFailPerSec: getCurrentFailPerSec(entry.NumFailures, entry.NumFailPerSec), + } + return +} + +// gauge vectors for requests +var ( + gaugeNumRequests = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "num_requests", + Help: "The number of requests", + }, + []string{"method", "name"}, + ) + gaugeNumFailures = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "num_failures", + Help: "The number of failures", + }, + []string{"method", "name"}, + ) + gaugeMedianResponseTime = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "median_response_time", + Help: "The median response time", + }, + []string{"method", "name"}, + ) + gaugeAverageResponseTime = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "average_response_time", + Help: "The average response time", + }, + []string{"method", "name"}, + ) + gaugeMinResponseTime = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "min_response_time", + Help: "The min response time", + }, + []string{"method", "name"}, + ) + gaugeMaxResponseTime = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "max_response_time", + Help: "The max response time", + }, + []string{"method", "name"}, + ) + gaugeAverageContentLength = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "average_content_length", + Help: "The average content length", + }, + []string{"method", "name"}, + ) + gaugeCurrentRPS = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "current_rps", + Help: "The current requests per second", + }, + []string{"method", "name"}, + ) + gaugeCurrentFailPerSec = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "current_fail_per_sec", + Help: "The current failure number per second", + }, + []string{"method", "name"}, + ) +) + +// gauges for total +var ( + gaugeUsers = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "users", + Help: "The current number of users", + }, + ) + gaugeTotalRPS = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "total_rps", + Help: "The requests per second in total", + }, + ) + gaugeTotalFailRatio = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "fail_ratio", + Help: "The ratio of request failures in total", + }, + ) + gaugeTransactionsPassed = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "transactions_passed", + Help: "The accumulated number of passed transactions", + }, + ) + gaugeTransactionsFailed = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "transactions_failed", + Help: "The accumulated number of failed transactions", + }, + ) +) + +// NewPrometheusPusherOutput returns a PrometheusPusherOutput. +func NewPrometheusPusherOutput(gatewayURL, jobName string) *PrometheusPusherOutput { + nodeUUID, _ := uuid.NewUUID() + return &PrometheusPusherOutput{ + pusher: push.New(gatewayURL, jobName).Grouping("instance", nodeUUID.String()), + } +} + +// PrometheusPusherOutput pushes boomer stats to Prometheus Pushgateway. +type PrometheusPusherOutput struct { + pusher *push.Pusher // Prometheus Pushgateway Pusher +} + +// OnStart will register all prometheus metric collectors +func (o *PrometheusPusherOutput) OnStart() { + log.Info().Msg("register prometheus metric collectors") + registry := prometheus.NewRegistry() + registry.MustRegister( + // gauge vectors for requests + gaugeNumRequests, + gaugeNumFailures, + gaugeMedianResponseTime, + gaugeAverageResponseTime, + gaugeMinResponseTime, + gaugeMaxResponseTime, + gaugeAverageContentLength, + gaugeCurrentRPS, + gaugeCurrentFailPerSec, + // gauges for total + gaugeUsers, + gaugeTotalRPS, + gaugeTotalFailRatio, + gaugeTransactionsPassed, + gaugeTransactionsFailed, + ) + o.pusher = o.pusher.Gatherer(registry) +} + +// OnStop of PrometheusPusherOutput has nothing to do. +func (o *PrometheusPusherOutput) OnStop() { + +} + +// OnEvent will push metric to Prometheus Pushgataway +func (o *PrometheusPusherOutput) OnEvent(data map[string]interface{}) { + output, err := convertData(data) + if err != nil { + log.Error().Err(err).Msg("failed to convert data") + return + } + + // user count + gaugeUsers.Set(float64(output.UserCount)) + + // rps in total + gaugeTotalRPS.Set(float64(output.TotalRPS)) + + // failure ratio in total + gaugeTotalFailRatio.Set(output.TotalFailRatio) + + // accumulated number of transactions + gaugeTransactionsPassed.Set(float64(output.TransactionsPassed)) + gaugeTransactionsFailed.Set(float64(output.TransactionsFailed)) + + for _, stat := range output.Stats { + method := stat.Method + name := stat.Name + gaugeNumRequests.WithLabelValues(method, name).Set(float64(stat.NumRequests)) + gaugeNumFailures.WithLabelValues(method, name).Set(float64(stat.NumFailures)) + gaugeMedianResponseTime.WithLabelValues(method, name).Set(float64(stat.medianResponseTime)) + gaugeAverageResponseTime.WithLabelValues(method, name).Set(float64(stat.avgResponseTime)) + gaugeMinResponseTime.WithLabelValues(method, name).Set(float64(stat.MinResponseTime)) + gaugeMaxResponseTime.WithLabelValues(method, name).Set(float64(stat.MaxResponseTime)) + gaugeAverageContentLength.WithLabelValues(method, name).Set(float64(stat.avgContentLength)) + gaugeCurrentRPS.WithLabelValues(method, name).Set(float64(stat.currentRps)) + gaugeCurrentFailPerSec.WithLabelValues(method, name).Set(float64(stat.currentFailPerSec)) + } + + if err := o.pusher.Push(); err != nil { + log.Error().Err(err).Msg("push to Pushgateway failed") + } +} diff --git a/internal/boomer/output_test.go b/internal/boomer/output_test.go new file mode 100644 index 00000000..76af41d1 --- /dev/null +++ b/internal/boomer/output_test.go @@ -0,0 +1,108 @@ +package boomer + +import ( + "math" + "testing" +) + +func TestGetMedianResponseTime(t *testing.T) { + numRequests := int64(10) + responseTimes := map[int64]int64{ + 100: 1, + 200: 3, + 300: 6, + } + + medianResponseTime := getMedianResponseTime(numRequests, responseTimes) + if medianResponseTime != 300 { + t.Error("medianResponseTime should be 300") + } + + responseTimes = map[int64]int64{} + + medianResponseTime = getMedianResponseTime(numRequests, responseTimes) + if medianResponseTime != 0 { + t.Error("medianResponseTime should be 0") + } +} + +func TestGetAvgResponseTime(t *testing.T) { + numRequests := int64(3) + totalResponseTime := int64(100) + + avgResponseTime := getAvgResponseTime(numRequests, totalResponseTime) + if math.Dim(float64(33.33), avgResponseTime) > 0.01 { + t.Error("avgResponseTime should be close to 33.33") + } + + avgResponseTime = getAvgResponseTime(int64(0), totalResponseTime) + if avgResponseTime != float64(0) { + t.Error("avgResponseTime should be close to 0") + } +} + +func TestGetAvgContentLength(t *testing.T) { + numRequests := int64(3) + totalContentLength := int64(100) + + avgContentLength := getAvgContentLength(numRequests, totalContentLength) + if avgContentLength != 33 { + t.Error("avgContentLength should be 33") + } + + avgContentLength = getAvgContentLength(int64(0), totalContentLength) + if avgContentLength != 0 { + t.Error("avgContentLength should be 0") + } +} + +func TestGetCurrentRps(t *testing.T) { + numRequests := int64(10) + numReqsPerSecond := map[int64]int64{} + + currentRps := getCurrentRps(numRequests, numReqsPerSecond) + if currentRps != 0 { + t.Error("currentRps should be 0") + } + + numReqsPerSecond[1] = 2 + numReqsPerSecond[2] = 3 + numReqsPerSecond[3] = 2 + numReqsPerSecond[4] = 3 + + currentRps = getCurrentRps(numRequests, numReqsPerSecond) + if currentRps != 2 { + t.Error("currentRps should be 2") + } +} + +func TestConsoleOutput(t *testing.T) { + o := NewConsoleOutput() + o.OnStart() + + data := map[string]interface{}{} + stat := map[string]interface{}{} + data["stats"] = []interface{}{stat} + + stat["name"] = "http" + stat["method"] = "post" + stat["num_requests"] = int64(100) + stat["num_failures"] = int64(10) + stat["response_times"] = map[int64]int64{ + 10: 1, + 100: 99, + } + stat["total_response_time"] = int64(9910) + stat["min_response_time"] = int64(10) + stat["max_response_time"] = int64(100) + stat["total_content_length"] = int64(100000) + stat["num_reqs_per_sec"] = map[int64]int64{ + 1: 20, + 2: 40, + 3: 40, + } + + o.OnEvent(data) + + o.OnStop() +} diff --git a/internal/boomer/ratelimiter.go b/internal/boomer/ratelimiter.go new file mode 100644 index 00000000..5cf98b9d --- /dev/null +++ b/internal/boomer/ratelimiter.go @@ -0,0 +1,213 @@ +package boomer + +import ( + "errors" + "math" + "strconv" + "strings" + "sync/atomic" + "time" +) + +// RateLimiter is used to put limits on task executions. +type RateLimiter interface { + // Start is used to enable the rate limiter. + // It can be implemented as a noop if not needed. + Start() + + // Acquire() is called before executing a task.Fn function. + // If Acquire() returns true, the task.Fn function will be executed. + // If Acquire() returns false, the task.Fn function won't be executed this time, but Acquire() will be called very soon. + // It works like: + // for { + // blocked := rateLimiter.Acquire() + // if !blocked { + // task.Fn() + // } + // } + // Acquire() should block the caller until execution is allowed. + Acquire() bool + + // Stop is used to disable the rate limiter. + // It can be implemented as a noop if not needed. + Stop() +} + +// A StableRateLimiter uses the token bucket algorithm. +// the bucket is refilled according to the refill period, no burst is allowed. +type StableRateLimiter struct { + threshold int64 + currentThreshold int64 + refillPeriod time.Duration + broadcastChannel chan bool + quitChannel chan bool +} + +// NewStableRateLimiter returns a StableRateLimiter. +func NewStableRateLimiter(threshold int64, refillPeriod time.Duration) (rateLimiter *StableRateLimiter) { + rateLimiter = &StableRateLimiter{ + threshold: threshold, + currentThreshold: threshold, + refillPeriod: refillPeriod, + broadcastChannel: make(chan bool), + } + return rateLimiter +} + +// Start to refill the bucket periodically. +func (limiter *StableRateLimiter) Start() { + limiter.quitChannel = make(chan bool) + quitChannel := limiter.quitChannel + go func() { + for { + select { + case <-quitChannel: + return + default: + atomic.StoreInt64(&limiter.currentThreshold, limiter.threshold) + time.Sleep(limiter.refillPeriod) + close(limiter.broadcastChannel) + limiter.broadcastChannel = make(chan bool) + } + } + }() +} + +// Acquire a token from the bucket, returns true if the bucket is exhausted. +func (limiter *StableRateLimiter) Acquire() (blocked bool) { + permit := atomic.AddInt64(&limiter.currentThreshold, -1) + if permit < 0 { + blocked = true + // block until the bucket is refilled + <-limiter.broadcastChannel + } else { + blocked = false + } + return blocked +} + +// Stop the rate limiter. +func (limiter *StableRateLimiter) Stop() { + close(limiter.quitChannel) +} + +// ErrParsingRampUpRate is the error returned if the format of rampUpRate is invalid. +var ErrParsingRampUpRate = errors.New("ratelimiter: invalid format of rampUpRate, try \"1\" or \"1/1s\"") + +// A RampUpRateLimiter uses the token bucket algorithm. +// the threshold is updated according to the warm up rate. +// the bucket is refilled according to the refill period, no burst is allowed. +type RampUpRateLimiter struct { + maxThreshold int64 + nextThreshold int64 + currentThreshold int64 + refillPeriod time.Duration + rampUpRate string + rampUpStep int64 + rampUpPeroid time.Duration + broadcastChannel chan bool + rampUpChannel chan bool + quitChannel chan bool +} + +// NewRampUpRateLimiter returns a RampUpRateLimiter. +// Valid formats of rampUpRate are "1", "1/1s". +func NewRampUpRateLimiter(maxThreshold int64, rampUpRate string, refillPeriod time.Duration) (rateLimiter *RampUpRateLimiter, err error) { + rateLimiter = &RampUpRateLimiter{ + maxThreshold: maxThreshold, + nextThreshold: 0, + currentThreshold: 0, + rampUpRate: rampUpRate, + refillPeriod: refillPeriod, + broadcastChannel: make(chan bool), + } + rateLimiter.rampUpStep, rateLimiter.rampUpPeroid, err = rateLimiter.parseRampUpRate(rateLimiter.rampUpRate) + if err != nil { + return nil, err + } + return rateLimiter, nil +} + +func (limiter *RampUpRateLimiter) parseRampUpRate(rampUpRate string) (rampUpStep int64, rampUpPeroid time.Duration, err error) { + if strings.Contains(rampUpRate, "/") { + tmp := strings.Split(rampUpRate, "/") + if len(tmp) != 2 { + return rampUpStep, rampUpPeroid, ErrParsingRampUpRate + } + rampUpStep, err := strconv.ParseInt(tmp[0], 10, 64) + if err != nil { + return rampUpStep, rampUpPeroid, ErrParsingRampUpRate + } + rampUpPeroid, err := time.ParseDuration(tmp[1]) + if err != nil { + return rampUpStep, rampUpPeroid, ErrParsingRampUpRate + } + return rampUpStep, rampUpPeroid, nil + } + + rampUpStep, err = strconv.ParseInt(rampUpRate, 10, 64) + if err != nil { + return rampUpStep, rampUpPeroid, ErrParsingRampUpRate + } + rampUpPeroid = time.Second + return rampUpStep, rampUpPeroid, nil +} + +// Start to refill the bucket periodically. +func (limiter *RampUpRateLimiter) Start() { + limiter.quitChannel = make(chan bool) + quitChannel := limiter.quitChannel + // bucket updater + go func() { + for { + select { + case <-quitChannel: + return + default: + atomic.StoreInt64(&limiter.currentThreshold, limiter.nextThreshold) + time.Sleep(limiter.refillPeriod) + close(limiter.broadcastChannel) + limiter.broadcastChannel = make(chan bool) + } + } + }() + // threshold updater + go func() { + for { + select { + case <-quitChannel: + return + default: + nextValue := limiter.nextThreshold + limiter.rampUpStep + if nextValue < 0 { + // int64 overflow + nextValue = int64(math.MaxInt64) + } + if nextValue > limiter.maxThreshold { + nextValue = limiter.maxThreshold + } + atomic.StoreInt64(&limiter.nextThreshold, nextValue) + time.Sleep(limiter.rampUpPeroid) + } + } + }() +} + +// Acquire a token from the bucket, returns true if the bucket is exhausted. +func (limiter *RampUpRateLimiter) Acquire() (blocked bool) { + permit := atomic.AddInt64(&limiter.currentThreshold, -1) + if permit < 0 { + blocked = true + // block until the bucket is refilled + <-limiter.broadcastChannel + } else { + blocked = false + } + return blocked +} + +// Stop the rate limiter. +func (limiter *RampUpRateLimiter) Stop() { + limiter.nextThreshold = 0 + close(limiter.quitChannel) +} diff --git a/internal/boomer/ratelimiter_test.go b/internal/boomer/ratelimiter_test.go new file mode 100644 index 00000000..0b8afafa --- /dev/null +++ b/internal/boomer/ratelimiter_test.go @@ -0,0 +1,101 @@ +package boomer + +import ( + "testing" + "time" +) + +func TestStableRateLimiter(t *testing.T) { + rateLimiter := NewStableRateLimiter(1, 10*time.Millisecond) + rateLimiter.Start() + defer rateLimiter.Stop() + + blocked := rateLimiter.Acquire() + if blocked { + t.Error("Unexpected blocked by rate limiter") + } + blocked = rateLimiter.Acquire() + if !blocked { + t.Error("Should be blocked") + } +} + +func TestRampUpRateLimiter(t *testing.T) { + rateLimiter, _ := NewRampUpRateLimiter(100, "10/200ms", 100*time.Millisecond) + rateLimiter.Start() + defer rateLimiter.Stop() + + time.Sleep(110 * time.Millisecond) + + for i := 0; i < 10; i++ { + blocked := rateLimiter.Acquire() + if blocked { + t.Error("Unexpected blocked by rate limiter") + } + } + blocked := rateLimiter.Acquire() + if !blocked { + t.Error("Should be blocked") + } + + time.Sleep(110 * time.Millisecond) + + // now, the threshold is 20 + for i := 0; i < 20; i++ { + blocked := rateLimiter.Acquire() + if blocked { + t.Error("Unexpected blocked by rate limiter") + } + } + blocked = rateLimiter.Acquire() + if !blocked { + t.Error("Should be blocked") + } +} + +func TestParseRampUpRate(t *testing.T) { + rateLimiter := &RampUpRateLimiter{} + rampUpStep, rampUpPeriod, _ := rateLimiter.parseRampUpRate("100") + if rampUpStep != 100 { + t.Error("Wrong rampUpStep, expected: 100, was:", rampUpStep) + } + if rampUpPeriod != time.Second { + t.Error("Wrong rampUpPeriod, expected: 1s, was:", rampUpPeriod) + } + rampUpStep, rampUpPeriod, _ = rateLimiter.parseRampUpRate("200/10s") + if rampUpStep != 200 { + t.Error("Wrong rampUpStep, expected: 200, was:", rampUpStep) + } + if rampUpPeriod != 10*time.Second { + t.Error("Wrong rampUpPeriod, expected: 10s, was:", rampUpPeriod) + } +} + +func TestParseInvalidRampUpRate(t *testing.T) { + rateLimiter := &RampUpRateLimiter{} + + _, _, err := rateLimiter.parseRampUpRate("A/1m") + if err == nil || err != ErrParsingRampUpRate { + t.Error("Expected ErrParsingRampUpRate") + } + + _, _, err = rateLimiter.parseRampUpRate("A") + if err == nil || err != ErrParsingRampUpRate { + t.Error("Expected ErrParsingRampUpRate") + } + + _, _, err = rateLimiter.parseRampUpRate("200/1s/") + if err == nil || err != ErrParsingRampUpRate { + t.Error("Expected ErrParsingRampUpRate") + } + + _, _, err = rateLimiter.parseRampUpRate("200/1") + if err == nil || err != ErrParsingRampUpRate { + t.Error("Expected ErrParsingRampUpRate") + } + + rateLimiter, err = NewRampUpRateLimiter(1, "200/1", time.Second) + if err == nil || err != ErrParsingRampUpRate { + t.Error("Expected ErrParsingRampUpRate") + } +} diff --git a/internal/boomer/runner.go b/internal/boomer/runner.go new file mode 100644 index 00000000..d3f11801 --- /dev/null +++ b/internal/boomer/runner.go @@ -0,0 +1,271 @@ +package boomer + +import ( + "fmt" + "math/rand" + "os" + "runtime/debug" + "sync" + "sync/atomic" + "time" + + "github.com/rs/zerolog/log" +) + +const ( + stateInit = "ready" + stateSpawning = "spawning" + stateRunning = "running" + stateStopped = "stopped" + stateQuitting = "quitting" +) + +const ( + reportStatsInterval = 3 * time.Second +) + +type runner struct { + state string + + tasks []*Task + totalTaskWeight int + + rateLimiter RateLimiter + rateLimitEnabled bool + stats *requestStats + + numClients int32 + spawnRate float64 + + // all running workers(goroutines) will select on this channel. + // close this channel will stop all running workers. + stopChan chan bool + + // close this channel will stop all goroutines used in runner. + closeChan chan bool + + outputs []Output +} + +// safeRun runs fn and recovers from unexpected panics. +// it prevents panics from Task.Fn crashing boomer. +func (r *runner) safeRun(fn func()) { + defer func() { + // don't panic + err := recover() + if err != nil { + stackTrace := debug.Stack() + errMsg := fmt.Sprintf("%v", err) + os.Stderr.Write([]byte(errMsg)) + os.Stderr.Write([]byte("\n")) + os.Stderr.Write(stackTrace) + } + }() + fn() +} + +func (r *runner) addOutput(o Output) { + r.outputs = append(r.outputs, o) +} + +func (r *runner) outputOnStart() { + size := len(r.outputs) + if size == 0 { + return + } + wg := sync.WaitGroup{} + wg.Add(size) + for _, output := range r.outputs { + go func(o Output) { + o.OnStart() + wg.Done() + }(output) + } + wg.Wait() +} + +func (r *runner) outputOnEevent(data map[string]interface{}) { + size := len(r.outputs) + if size == 0 { + return + } + wg := sync.WaitGroup{} + wg.Add(size) + for _, output := range r.outputs { + go func(o Output) { + o.OnEvent(data) + wg.Done() + }(output) + } + wg.Wait() +} + +func (r *runner) outputOnStop() { + size := len(r.outputs) + if size == 0 { + return + } + wg := sync.WaitGroup{} + wg.Add(size) + for _, output := range r.outputs { + go func(o Output) { + o.OnStop() + wg.Done() + }(output) + } + wg.Wait() +} + +func (r *runner) spawnWorkers(spawnCount int, quit chan bool, spawnCompleteFunc func()) { + log.Info().Int("spawnCount", spawnCount).Msg("Spawning clients immediately") + + for i := 1; i <= spawnCount; i++ { + select { + case <-quit: + // quit spawning goroutine + return + default: + atomic.AddInt32(&r.numClients, 1) + go func() { + for { + select { + case <-quit: + return + default: + if r.rateLimitEnabled { + blocked := r.rateLimiter.Acquire() + if !blocked { + task := r.getTask() + r.safeRun(task.Fn) + } + } else { + task := r.getTask() + r.safeRun(task.Fn) + } + } + } + }() + } + } + + if spawnCompleteFunc != nil { + spawnCompleteFunc() + } +} + +// setTasks will set the runner's task list AND the total task weight +// which is used to get a random task later +func (r *runner) setTasks(t []*Task) { + r.tasks = t + + weightSum := 0 + for _, task := range r.tasks { + weightSum += task.Weight + } + r.totalTaskWeight = weightSum +} + +func (r *runner) getTask() *Task { + tasksCount := len(r.tasks) + if tasksCount == 1 { + // Fast path + return r.tasks[0] + } + + rs := rand.New(rand.NewSource(time.Now().UnixNano())) + + totalWeight := r.totalTaskWeight + if totalWeight <= 0 { + // If all the tasks have not weights defined, they have the same chance to run + randNum := rs.Intn(tasksCount) + return r.tasks[randNum] + } + + randNum := rs.Intn(totalWeight) + runningSum := 0 + for _, task := range r.tasks { + runningSum += task.Weight + if runningSum > randNum { + return task + } + } + + return nil +} + +func (r *runner) startSpawning(spawnCount int, spawnRate float64, spawnCompleteFunc func()) { + r.stats.clearStatsChan <- true + r.stopChan = make(chan bool) + + r.numClients = 0 + + go r.spawnWorkers(spawnCount, r.stopChan, spawnCompleteFunc) +} + +func (r *runner) stop() { + // stop previous goroutines without blocking + // those goroutines will exit when r.safeRun returns + close(r.stopChan) + if r.rateLimitEnabled { + r.rateLimiter.Stop() + } +} + +type localRunner struct { + runner + + spawnCount int +} + +func newLocalRunner(tasks []*Task, rateLimiter RateLimiter, spawnCount int, spawnRate float64) (r *localRunner) { + r = &localRunner{} + r.setTasks(tasks) + r.spawnRate = spawnRate + r.spawnCount = spawnCount + r.closeChan = make(chan bool) + + if rateLimiter != nil { + r.rateLimitEnabled = true + r.rateLimiter = rateLimiter + } + + r.stats = newRequestStats() + return r +} + +func (r *localRunner) run() { + r.state = stateInit + r.stats.start() + r.outputOnStart() + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + for { + select { + case data := <-r.stats.messageToRunnerChan: + data["user_count"] = r.numClients + r.outputOnEevent(data) + case <-r.closeChan: + r.stop() + wg.Done() + r.outputOnStop() + return + } + } + }() + + if r.rateLimitEnabled { + r.rateLimiter.Start() + } + r.startSpawning(r.spawnCount, r.spawnRate, nil) + + wg.Wait() +} + +func (r *localRunner) close() { + if r.stats != nil { + r.stats.close() + } + close(r.closeChan) +} diff --git a/internal/boomer/runner_test.go b/internal/boomer/runner_test.go new file mode 100644 index 00000000..e1a7e3f0 --- /dev/null +++ b/internal/boomer/runner_test.go @@ -0,0 +1,91 @@ +package boomer + +import ( + "testing" + "time" +) + +type HitOutput struct { + onStart bool + onEvent bool + onStop bool +} + +func (o *HitOutput) OnStart() { + o.onStart = true +} + +func (o *HitOutput) OnEvent(data map[string]interface{}) { + o.onEvent = true +} + +func (o *HitOutput) OnStop() { + o.onStop = true +} + +func TestSafeRun(t *testing.T) { + runner := &runner{} + runner.safeRun(func() { + panic("Runner will catch this panic") + }) +} + +func TestOutputOnStart(t *testing.T) { + hitOutput := &HitOutput{} + hitOutput2 := &HitOutput{} + runner := &runner{} + runner.addOutput(hitOutput) + runner.addOutput(hitOutput2) + runner.outputOnStart() + if !hitOutput.onStart { + t.Error("hitOutput's OnStart has not been called") + } + if !hitOutput2.onStart { + t.Error("hitOutput2's OnStart has not been called") + } +} + +func TestOutputOnEevent(t *testing.T) { + hitOutput := &HitOutput{} + hitOutput2 := &HitOutput{} + runner := &runner{} + runner.addOutput(hitOutput) + runner.addOutput(hitOutput2) + runner.outputOnEevent(nil) + if !hitOutput.onEvent { + t.Error("hitOutput's OnEvent has not been called") + } + if !hitOutput2.onEvent { + t.Error("hitOutput2's OnEvent has not been called") + } +} + +func TestOutputOnStop(t *testing.T) { + hitOutput := &HitOutput{} + hitOutput2 := &HitOutput{} + runner := &runner{} + runner.addOutput(hitOutput) + runner.addOutput(hitOutput2) + runner.outputOnStop() + if !hitOutput.onStop { + t.Error("hitOutput's OnStop has not been called") + } + if !hitOutput2.onStop { + t.Error("hitOutput2's OnStop has not been called") + } +} + +func TestLocalRunner(t *testing.T) { + taskA := &Task{ + Weight: 10, + Fn: func() { + time.Sleep(time.Second) + }, + Name: "TaskA", + } + tasks := []*Task{taskA} + runner := newLocalRunner(tasks, nil, 2, 2) + go runner.run() + time.Sleep(4 * time.Second) + runner.close() +} diff --git a/internal/boomer/stats.go b/internal/boomer/stats.go new file mode 100644 index 00000000..9978a584 --- /dev/null +++ b/internal/boomer/stats.go @@ -0,0 +1,351 @@ +package boomer + +import ( + "encoding/json" + "time" +) + +type transaction struct { + name string + success bool + elapsedTime int64 + contentSize int64 +} + +type requestSuccess struct { + requestType string + name string + responseTime int64 + responseLength int64 +} + +type requestFailure struct { + requestType string + name string + responseTime int64 + errMsg string +} + +type requestStats struct { + entries map[string]*statsEntry + errors map[string]*statsError + total *statsEntry + startTime int64 + + transactionChan chan *transaction + transactionPassed int64 // accumulated number of passed transactions + transactionFailed int64 // accumulated number of failed transactions + + requestSuccessChan chan *requestSuccess + requestFailureChan chan *requestFailure + clearStatsChan chan bool + messageToRunnerChan chan map[string]interface{} + shutdownChan chan bool +} + +func newRequestStats() (stats *requestStats) { + entries := make(map[string]*statsEntry) + errors := make(map[string]*statsError) + + stats = &requestStats{ + entries: entries, + errors: errors, + } + stats.transactionChan = make(chan *transaction, 100) + stats.requestSuccessChan = make(chan *requestSuccess, 100) + stats.requestFailureChan = make(chan *requestFailure, 100) + stats.clearStatsChan = make(chan bool) + stats.messageToRunnerChan = make(chan map[string]interface{}, 10) + stats.shutdownChan = make(chan bool) + + stats.total = &statsEntry{ + Name: "Total", + Method: "", + } + stats.total.reset() + + return stats +} + +func (s *requestStats) logTransaction(name string, success bool, responseTime int64, contentLength int64) { + if success { + s.transactionPassed++ + } else { + s.transactionFailed++ + } + s.get(name, "transaction").log(responseTime, contentLength) +} + +func (s *requestStats) logRequest(method, name string, responseTime int64, contentLength int64) { + s.total.log(responseTime, contentLength) + s.get(name, method).log(responseTime, contentLength) +} + +func (s *requestStats) logError(method, name, err string) { + s.total.logError(err) + s.get(name, method).logError(err) + + // store error in errors map + key := genMD5(method, name, err) + entry, ok := s.errors[key] + if !ok { + entry = &statsError{ + name: name, + method: method, + error: err, + } + s.errors[key] = entry + } + entry.occured() +} + +func (s *requestStats) get(name string, method string) (entry *statsEntry) { + entry, ok := s.entries[name+method] + if !ok { + newEntry := &statsEntry{ + Name: name, + Method: method, + NumReqsPerSec: make(map[int64]int64), + ResponseTimes: make(map[int64]int64), + } + newEntry.reset() + s.entries[name+method] = newEntry + return newEntry + } + return entry +} + +func (s *requestStats) clearAll() { + s.total = &statsEntry{ + Name: "Total", + Method: "", + } + s.total.reset() + + s.transactionPassed = 0 + s.transactionFailed = 0 + s.entries = make(map[string]*statsEntry) + s.errors = make(map[string]*statsError) + s.startTime = time.Now().Unix() +} + +func (s *requestStats) serializeStats() []interface{} { + entries := make([]interface{}, 0, len(s.entries)) + for _, v := range s.entries { + if !(v.NumRequests == 0 && v.NumFailures == 0) { + entries = append(entries, v.getStrippedReport()) + } + } + return entries +} + +func (s *requestStats) serializeErrors() map[string]map[string]interface{} { + errors := make(map[string]map[string]interface{}) + for k, v := range s.errors { + errors[k] = v.toMap() + } + return errors +} + +func (s *requestStats) collectReportData() map[string]interface{} { + data := make(map[string]interface{}) + data["transactions"] = map[string]int64{ + "passed": s.transactionPassed, + "failed": s.transactionFailed, + } + data["stats"] = s.serializeStats() + data["stats_total"] = s.total.getStrippedReport() + data["errors"] = s.serializeErrors() + s.errors = make(map[string]*statsError) + return data +} + +func (s *requestStats) start() { + go func() { + var ticker = time.NewTicker(reportStatsInterval) + for { + select { + case t := <-s.transactionChan: + s.logTransaction(t.name, t.success, t.elapsedTime, t.contentSize) + case m := <-s.requestSuccessChan: + s.logRequest(m.requestType, m.name, m.responseTime, m.responseLength) + case n := <-s.requestFailureChan: + s.logRequest(n.requestType, n.name, n.responseTime, 0) + s.logError(n.requestType, n.name, n.errMsg) + case <-s.clearStatsChan: + s.clearAll() + case <-ticker.C: + data := s.collectReportData() + // send data to channel, no network IO in this goroutine + s.messageToRunnerChan <- data + case <-s.shutdownChan: + return + } + } + }() +} + +// close is used by unit tests to avoid leakage of goroutines +func (s *requestStats) close() { + close(s.shutdownChan) +} + +// statsEntry represents a single stats entry (name and method) +type statsEntry struct { + // Name (URL) of this stats entry + Name string `json:"name"` + // Method (GET, POST, PUT, etc.) + Method string `json:"method"` + // The number of requests made + NumRequests int64 `json:"num_requests"` + // Number of failed request + NumFailures int64 `json:"num_failures"` + // Total sum of the response times + TotalResponseTime int64 `json:"total_response_time"` + // Minimum response time + MinResponseTime int64 `json:"min_response_time"` + // Maximum response time + MaxResponseTime int64 `json:"max_response_time"` + // A {second => request_count} dict that holds the number of requests made per second + NumReqsPerSec map[int64]int64 `json:"num_reqs_per_sec"` + // A (second => failure_count) dict that hold the number of failures per second + NumFailPerSec map[int64]int64 `json:"num_fail_per_sec"` + // A {response_time => count} dict that holds the response time distribution of all the requests + // The keys (the response time in ms) are rounded to store 1, 2, ... 9, 10, 20. .. 90, + // 100, 200 .. 900, 1000, 2000 ... 9000, in order to save memory. + // This dict is used to calculate the median and percentile response times. + ResponseTimes map[int64]int64 `json:"response_times"` + // The sum of the content length of all the requests for this entry + TotalContentLength int64 `json:"total_content_length"` + // Time of the first request for this entry + StartTime int64 `json:"start_time"` + // Time of the last request for this entry + LastRequestTimestamp int64 `json:"last_request_timestamp"` + // Boomer doesn't allow None response time for requests like locust. + // num_none_requests is added to keep compatible with locust. + NumNoneRequests int64 `json:"num_none_requests"` +} + +func (s *statsEntry) reset() { + s.StartTime = time.Now().Unix() + s.NumRequests = 0 + s.NumFailures = 0 + s.TotalResponseTime = 0 + s.ResponseTimes = make(map[int64]int64) + s.MinResponseTime = 0 + s.MaxResponseTime = 0 + s.LastRequestTimestamp = time.Now().Unix() + s.NumReqsPerSec = make(map[int64]int64) + s.NumFailPerSec = make(map[int64]int64) + s.TotalContentLength = 0 +} + +func (s *statsEntry) log(responseTime int64, contentLength int64) { + s.NumRequests++ + + s.logTimeOfRequest() + s.logResponseTime(responseTime) + + s.TotalContentLength += contentLength +} + +func (s *statsEntry) logTimeOfRequest() { + key := time.Now().Unix() + _, ok := s.NumReqsPerSec[key] + if !ok { + s.NumReqsPerSec[key] = 1 + } else { + s.NumReqsPerSec[key]++ + } + + s.LastRequestTimestamp = key +} + +func (s *statsEntry) logResponseTime(responseTime int64) { + s.TotalResponseTime += responseTime + + if s.MinResponseTime == 0 { + s.MinResponseTime = responseTime + } + + if responseTime < s.MinResponseTime { + s.MinResponseTime = responseTime + } + + if responseTime > s.MaxResponseTime { + s.MaxResponseTime = responseTime + } + + var roundedResponseTime int64 + + // to avoid too much data that has to be transferred to the master node when + // running in distributed mode, we save the response time rounded in a dict + // so that 147 becomes 150, 3432 becomes 3400 and 58760 becomes 59000 + // see also locust's stats.py + if responseTime < 100 { + roundedResponseTime = responseTime + } else if responseTime < 1000 { + roundedResponseTime = int64(round(float64(responseTime), .5, -1)) + } else if responseTime < 10000 { + roundedResponseTime = int64(round(float64(responseTime), .5, -2)) + } else { + roundedResponseTime = int64(round(float64(responseTime), .5, -3)) + } + + _, ok := s.ResponseTimes[roundedResponseTime] + if !ok { + s.ResponseTimes[roundedResponseTime] = 1 + } else { + s.ResponseTimes[roundedResponseTime]++ + } +} + +func (s *statsEntry) logError(err string) { + s.NumFailures++ + key := time.Now().Unix() + _, ok := s.NumFailPerSec[key] + if !ok { + s.NumFailPerSec[key] = 1 + } else { + s.NumFailPerSec[key]++ + } +} + +func (s *statsEntry) serialize() map[string]interface{} { + var result map[string]interface{} + val, err := json.Marshal(s) + if err != nil { + return nil + } + err = json.Unmarshal(val, &result) + if err != nil { + return nil + } + return result +} + +func (s *statsEntry) getStrippedReport() map[string]interface{} { + report := s.serialize() + s.reset() + return report +} + +type statsError struct { + name string + method string + error string + occurrences int64 +} + +func (err *statsError) occured() { + err.occurrences++ +} + +func (err *statsError) toMap() map[string]interface{} { + m := make(map[string]interface{}) + m["method"] = err.method + m["name"] = err.name + m["error"] = err.error + m["occurrences"] = err.occurrences + return m +} diff --git a/internal/boomer/stats_test.go b/internal/boomer/stats_test.go new file mode 100644 index 00000000..4c82b76c --- /dev/null +++ b/internal/boomer/stats_test.go @@ -0,0 +1,250 @@ +package boomer + +import ( + "testing" + "time" +) + +func TestLogRequest(t *testing.T) { + newStats := newRequestStats() + newStats.logRequest("http", "success", 2, 30) + newStats.logRequest("http", "success", 3, 40) + newStats.logRequest("http", "success", 2, 40) + newStats.logRequest("http", "success", 1, 20) + entry := newStats.get("success", "http") + + if entry.NumRequests != 4 { + t.Error("numRequests is wrong, expected: 4, got:", entry.NumRequests) + } + if entry.MinResponseTime != 1 { + t.Error("minResponseTime is wrong, expected: 1, got:", entry.MinResponseTime) + } + if entry.MaxResponseTime != 3 { + t.Error("maxResponseTime is wrong, expected: 3, got:", entry.MaxResponseTime) + } + if entry.TotalResponseTime != 8 { + t.Error("totalResponseTime is wrong, expected: 8, got:", entry.TotalResponseTime) + } + if entry.TotalContentLength != 130 { + t.Error("totalContentLength is wrong, expected: 130, got:", entry.TotalContentLength) + } + + // check newStats.total + if newStats.total.NumRequests != 4 { + t.Error("newStats.total.numRequests is wrong, expected: 4, got:", newStats.total.NumRequests) + } + if newStats.total.MinResponseTime != 1 { + t.Error("newStats.total.minResponseTime is wrong, expected: 1, got:", newStats.total.MinResponseTime) + } + if newStats.total.MaxResponseTime != 3 { + t.Error("newStats.total.maxResponseTime is wrong, expected: 3, got:", newStats.total.MaxResponseTime) + } + if newStats.total.TotalResponseTime != 8 { + t.Error("newStats.total.totalResponseTime is wrong, expected: 8, got:", newStats.total.TotalResponseTime) + } + if newStats.total.TotalContentLength != 130 { + t.Error("newStats.total.totalContentLength is wrong, expected: 130, got:", newStats.total.TotalContentLength) + } +} + +func BenchmarkLogRequest(b *testing.B) { + newStats := newRequestStats() + for i := 0; i < b.N; i++ { + newStats.logRequest("http", "success", 2, 30) + } +} + +func TestRoundedResponseTime(t *testing.T) { + newStats := newRequestStats() + newStats.logRequest("http", "success", 147, 1) + newStats.logRequest("http", "success", 3432, 1) + newStats.logRequest("http", "success", 58760, 1) + entry := newStats.get("success", "http") + responseTimes := entry.ResponseTimes + + if len(responseTimes) != 3 { + t.Error("len(responseTimes) is wrong, expected: 3, got:", len(responseTimes)) + } + + if val, ok := responseTimes[150]; !ok || val != 1 { + t.Error("Rounded response time should be", 150) + } + + if val, ok := responseTimes[3400]; !ok || val != 1 { + t.Error("Rounded response time should be", 3400) + } + + if val, ok := responseTimes[59000]; !ok || val != 1 { + t.Error("Rounded response time should be", 59000) + } +} + +func TestLogError(t *testing.T) { + newStats := newRequestStats() + newStats.logError("http", "failure", "500 error") + newStats.logError("http", "failure", "400 error") + newStats.logError("http", "failure", "400 error") + entry := newStats.get("failure", "http") + + if entry.NumFailures != 3 { + t.Error("numFailures is wrong, expected: 3, got:", entry.NumFailures) + } + + if newStats.total.NumFailures != 3 { + t.Error("newStats.total.numFailures is wrong, expected: 3, got:", newStats.total.NumFailures) + } + + // md5("httpfailure500 error") = 547c38e4e4742c1c581f9e2809ba4f55 + err500 := newStats.errors["547c38e4e4742c1c581f9e2809ba4f55"] + if err500.error != "500 error" { + t.Error("Error message is wrong, expected: 500 error, got:", err500.error) + } + if err500.occurrences != 1 { + t.Error("Error occurrences is wrong, expected: 1, got:", err500.occurrences) + } + + // md5("httpfailure400 error") = f391c310401ad8e10e929f2ee1a614e4 + err400 := newStats.errors["f391c310401ad8e10e929f2ee1a614e4"] + if err400.error != "400 error" { + t.Error("Error message is wrong, expected: 400 error, got:", err400.error) + } + if err400.occurrences != 2 { + t.Error("Error occurrences is wrong, expected: 2, got:", err400.occurrences) + } + +} + +func BenchmarkLogError(b *testing.B) { + newStats := newRequestStats() + for i := 0; i < b.N; i++ { + // LogError use md5 to calculate hash keys, it may slow down the only goroutine, + // which consumes both requestSuccessChannel and requestFailureChannel. + newStats.logError("http", "failure", "500 error") + } +} + +func TestClearAll(t *testing.T) { + newStats := newRequestStats() + newStats.logRequest("http", "success", 1, 20) + newStats.clearAll() + + if newStats.total.NumRequests != 0 { + t.Error("After clearAll(), newStats.total.numRequests is wrong, expected: 0, got:", newStats.total.NumRequests) + } +} + +func TestClearAllByChannel(t *testing.T) { + newStats := newRequestStats() + newStats.start() + defer newStats.close() + newStats.logRequest("http", "success", 1, 20) + newStats.clearStatsChan <- true + + if newStats.total.NumRequests != 0 { + t.Error("After clearAll(), newStats.total.numRequests is wrong, expected: 0, got:", newStats.total.NumRequests) + } +} + +func TestSerializeStats(t *testing.T) { + newStats := newRequestStats() + newStats.logRequest("http", "success", 1, 20) + + serialized := newStats.serializeStats() + if len(serialized) != 1 { + t.Error("The length of serialized results is wrong, expected: 1, got:", len(serialized)) + return + } + + first := serialized[0] + entry, err := deserializeStatsEntry(first) + if err != nil { + t.Fail() + } + + if entry.Name != "success" { + t.Error("The name is wrong, expected:", "success", "got:", entry.Name) + } + if entry.Method != "http" { + t.Error("The method is wrong, expected:", "http", "got:", entry.Method) + } + if entry.NumRequests != int64(1) { + t.Error("The num_requests is wrong, expected:", 1, "got:", entry.NumRequests) + } + if entry.NumFailures != int64(0) { + t.Error("The num_failures is wrong, expected:", 0, "got:", entry.NumFailures) + } +} + +func TestSerializeErrors(t *testing.T) { + newStats := newRequestStats() + newStats.logError("http", "failure", "500 error") + newStats.logError("http", "failure", "400 error") + newStats.logError("http", "failure", "400 error") + serialized := newStats.serializeErrors() + + if len(serialized) != 2 { + t.Error("The length of serialized results is wrong, expected: 2, got:", len(serialized)) + return + } + + for key, value := range serialized { + if key == "f391c310401ad8e10e929f2ee1a614e4" { + err := value["error"].(string) + if err != "400 error" { + t.Error("expected: 400 error, got:", err) + } + occurrences := value["occurrences"].(int64) + if occurrences != int64(2) { + t.Error("expected: 2, got:", occurrences) + } + } + } +} + +func TestCollectReportData(t *testing.T) { + newStats := newRequestStats() + newStats.logRequest("http", "success", 2, 30) + newStats.logError("http", "failure", "500 error") + result := newStats.collectReportData() + + if _, ok := result["stats"]; !ok { + t.Error("Key stats not found") + } + if _, ok := result["stats_total"]; !ok { + t.Error("Key stats not found") + } + if _, ok := result["errors"]; !ok { + t.Error("Key stats not found") + } +} + +func TestStatsStart(t *testing.T) { + newStats := newRequestStats() + newStats.start() + defer newStats.close() + + newStats.requestSuccessChan <- &requestSuccess{ + requestType: "http", + name: "success", + responseTime: 2, + responseLength: 30, + } + + newStats.requestFailureChan <- &requestFailure{ + requestType: "http", + name: "failure", + responseTime: 1, + errMsg: "500 error", + } + + var ticker = time.NewTicker(reportStatsInterval + 500*time.Millisecond) + for { + select { + case <-ticker.C: + t.Error("Timeout waiting for stats reports to runner") + case <-newStats.messageToRunnerChan: + goto end + } + } +end: +} diff --git a/internal/boomer/task.go b/internal/boomer/task.go new file mode 100644 index 00000000..e913d093 --- /dev/null +++ b/internal/boomer/task.go @@ -0,0 +1,13 @@ +package boomer + +// Task is like the "Locust object" in locust, the python version. +// When boomer receives a start message from master, it will spawn several goroutines to run Task.Fn. +// But users can keep some information in the python version, they can't do the same things in boomer. +// Because Task.Fn is a pure function. +type Task struct { + // The weight is used to distribute goroutines over multiple tasks. + Weight int + // Fn is called by the goroutines allocated to this task, in a loop. + Fn func() + Name string +} diff --git a/internal/boomer/utils.go b/internal/boomer/utils.go new file mode 100644 index 00000000..7d7bfe6f --- /dev/null +++ b/internal/boomer/utils.go @@ -0,0 +1,77 @@ +package boomer + +import ( + "crypto/md5" + "fmt" + "io" + "math" + "os" + "runtime/pprof" + "time" + + "github.com/rs/zerolog/log" +) + +func round(val float64, roundOn float64, places int) (newVal float64) { + var round float64 + pow := math.Pow(10, float64(places)) + digit := pow * val + _, div := math.Modf(digit) + if div >= roundOn { + round = math.Ceil(digit) + } else { + round = math.Floor(digit) + } + newVal = round / pow + return +} + +// genMD5 returns the md5 hash of strings. +func genMD5(slice ...string) string { + h := md5.New() + for _, v := range slice { + io.WriteString(h, v) + } + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// startMemoryProfile starts memory profiling and save the results in file. +func startMemoryProfile(file string, duration time.Duration) (err error) { + f, err := os.Create(file) + if err != nil { + return err + } + + log.Info().Dur("duration", duration).Msg("Start memory profiling") + time.AfterFunc(duration, func() { + err = pprof.WriteHeapProfile(f) + if err != nil { + log.Error().Err(err).Msg("failed to write memory profile") + } + f.Close() + log.Info().Dur("duration", duration).Msg("Stop memory profiling") + }) + return nil +} + +// startCPUProfile starts cpu profiling and save the results in file. +func startCPUProfile(file string, duration time.Duration) (err error) { + f, err := os.Create(file) + if err != nil { + return err + } + + log.Info().Dur("duration", duration).Msg("Start CPU profiling") + err = pprof.StartCPUProfile(f) + if err != nil { + f.Close() + return err + } + + time.AfterFunc(duration, func() { + pprof.StopCPUProfile() + f.Close() + log.Info().Dur("duration", duration).Msg("Stop CPU profiling") + }) + return nil +} diff --git a/internal/boomer/utils_test.go b/internal/boomer/utils_test.go new file mode 100644 index 00000000..c56d1457 --- /dev/null +++ b/internal/boomer/utils_test.go @@ -0,0 +1,73 @@ +package boomer + +import ( + "os" + "testing" + "time" +) + +func TestRound(t *testing.T) { + if int(round(float64(147.5002), .5, -1)) != 150 { + t.Error("147.5002 should be rounded to 150") + } + + if int(round(float64(3432.5002), .5, -2)) != 3400 { + t.Error("3432.5002 should be rounded to 3400") + } + + roundOne := round(float64(58760.5002), .5, -3) + roundTwo := round(float64(58960.6003), .5, -3) + if roundOne != roundTwo { + t.Error("round(58760.5002) should be equal to round(58960.6003)") + } + + roundOne = round(float64(58360.5002), .5, -3) + roundTwo = round(float64(58460.6003), .5, -3) + if roundOne != roundTwo { + t.Error("round(58360.5002) should be equal to round(58460.6003)") + } + + roundOne = round(float64(58360), .5, -3) + roundTwo = round(float64(58460), .5, -3) + if roundOne != roundTwo { + t.Error("round(58360) should be equal to round(58460)") + } + +} + +func TestGenMD5(t *testing.T) { + hashValue := genMD5("Hello", "World!") + if hashValue != "06e0e6637d27b2622ab52022db713ce2" { + t.Error("Expected: 06e0e6637d27b2622ab52022db713ce2, Got: ", hashValue) + } +} + +func TestStartMemoryProfile(t *testing.T) { + if _, err := os.Stat("mem.pprof"); os.IsExist(err) { + os.Remove("mem.pprof") + } + if err := startMemoryProfile("mem.pprof", 2*time.Second); err != nil { + t.Error("Error starting memory profiling") + } + time.Sleep(2100 * time.Millisecond) + if _, err := os.Stat("mem.pprof"); os.IsNotExist(err) { + t.Error("File mem.pprof is not generated") + } else { + os.Remove("mem.pprof") + } +} + +func TestStartCPUProfile(t *testing.T) { + if _, err := os.Stat("cpu.pprof"); os.IsExist(err) { + os.Remove("cpu.pprof") + } + if err := startCPUProfile("cpu.pprof", 2*time.Second); err != nil { + t.Error("Error starting cpu profiling") + } + time.Sleep(2100 * time.Millisecond) + if _, err := os.Stat("cpu.pprof"); os.IsNotExist(err) { + t.Error("File cpu.pprof is not generated") + } else { + os.Remove("cpu.pprof") + } +} diff --git a/internal/ga/client_test.go b/internal/ga/client_test.go index e0479611..a1e1cae4 100644 --- a/internal/ga/client_test.go +++ b/internal/ga/client_test.go @@ -20,11 +20,11 @@ func TestStructToUrlValues(t *testing.T) { event := EventTracking{ Category: "unittest", Action: "convert", - Label: "StructToUrlValues", + Label: "v0.3.0", Value: "123", } val := structToUrlValues(event) - if val.Encode() != "ea=convert&ec=unittest&el=StructToUrlValues&ev=123" { + if val.Encode() != "ea=convert&ec=unittest&el=v0.3.0&ev=123" { t.Fail() } } diff --git a/internal/ga/events.go b/internal/ga/events.go index 71f62b1c..2044d196 100644 --- a/internal/ga/events.go +++ b/internal/ga/events.go @@ -4,6 +4,8 @@ import ( "fmt" "net/url" "time" + + "github.com/httprunner/hrp/internal/version" ) type IEvent interface { @@ -14,7 +16,7 @@ type EventTracking struct { HitType string `form:"t"` // Event hit type = event Category string `form:"ec"` // Required. Event Category. Action string `form:"ea"` // Required. Event Action. - Label string `form:"el"` // Optional. Event label + Label string `form:"el"` // Optional. Event label, used as version. Value string `form:"ev"` // Optional. Event value, must be digits, "123" } @@ -30,6 +32,7 @@ func (e EventTracking) StartTiming(variable string) UserTimingTracking { func (e EventTracking) ToUrlValues() url.Values { e.HitType = "event" + e.Label = version.VERSION return structToUrlValues(e) } @@ -45,6 +48,7 @@ type UserTimingTracking struct { func (e UserTimingTracking) ToUrlValues() url.Values { e.HitType = "timing" + e.Label = version.VERSION e.Duration = fmt.Sprintf("%d", int64(e.duration.Seconds()*1000)) return structToUrlValues(e) } diff --git a/internal/version/init.go b/internal/version/init.go index b62e2970..a29e664d 100644 --- a/internal/version/init.go +++ b/internal/version/init.go @@ -1,3 +1,3 @@ package version -const VERSION = "v0.2.2" +const VERSION = "v0.3.0" diff --git a/log.go b/log.go deleted file mode 100644 index cdd3867f..00000000 --- a/log.go +++ /dev/null @@ -1,47 +0,0 @@ -package hrp - -import ( - "os" - "strings" - - "github.com/rs/zerolog" - zlog "github.com/rs/zerolog/log" -) - -var log = zlog.Logger - -// SetLogger configures the log level and format. -func SetLogger(level string, logJSON bool) { - if !logJSON { - setLogPretty() - } - setLogLevel(level) -} - -func setLogLevel(level string) { - level = strings.ToUpper(level) - log.Info().Msgf("Set log level to %s", level) - switch level { - case "DEBUG": - zerolog.SetGlobalLevel(zerolog.DebugLevel) - case "INFO": - zerolog.SetGlobalLevel(zerolog.InfoLevel) - case "WARN": - zerolog.SetGlobalLevel(zerolog.WarnLevel) - case "ERROR": - zerolog.SetGlobalLevel(zerolog.ErrorLevel) - case "FATAL": - zerolog.SetGlobalLevel(zerolog.FatalLevel) - case "PANIC": - zerolog.SetGlobalLevel(zerolog.PanicLevel) - } -} - -func setLogPretty() { - log = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) - log.Info().Msg("Set log to color console other than JSON format.") -} - -func GetLogger() zerolog.Logger { - return log -} diff --git a/models.go b/models.go index 20f727e7..cbaf4b4e 100644 --- a/models.go +++ b/models.go @@ -50,6 +50,8 @@ type TStep struct { Name string `json:"name" yaml:"name"` // required Request *Request `json:"request,omitempty" yaml:"request,omitempty"` TestCase *TestCase `json:"testcase,omitempty" yaml:"testcase,omitempty"` + Transaction *Transaction `json:"transaction,omitempty" yaml:"transaction,omitempty"` + Rendezvous *Rendezvous `json:"rendezvous,omitempty" yaml:"rendezvous,omitempty"` Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"` SetupHooks []string `json:"setup_hooks,omitempty" yaml:"setup_hooks,omitempty"` TeardownHooks []string `json:"teardown_hooks,omitempty" yaml:"teardown_hooks,omitempty"` @@ -58,6 +60,33 @@ type TStep struct { Export []string `json:"export,omitempty" yaml:"export,omitempty"` } +type stepType string + +const ( + stepTypeRequest stepType = "request" + stepTypeTestCase stepType = "testcase" + stepTypeTransaction stepType = "transaction" + stepTypeRendezvous stepType = "rendezvous" +) + +type TransactionType string + +const ( + TransactionStart TransactionType = "start" + TransactionEnd TransactionType = "end" +) + +type Transaction struct { + Name string `json:"name" yaml:"name"` + Type TransactionType `json:"type" yaml:"type"` +} +type Rendezvous struct { + Name string `json:"name" yaml:"name"` // required + Percent float32 `json:"percent,omitempty" yaml:"percent,omitempty"` // default to 1(100%) + Number int64 `json:"number,omitempty" yaml:"number,omitempty"` + Timeout int64 `json:"timeout,omitempty" yaml:"timeout,omitempty"` // milliseconds +} + // TCase represents testcase data structure. // Each testcase includes one public config and several sequential teststeps. type TCase struct { @@ -65,23 +94,34 @@ type TCase struct { TestSteps []*TStep `json:"teststeps" yaml:"teststeps"` } -// IStep represents interface for all types for teststeps. +// IConfig represents interface for testcase config, +// includes Config. +type IConfig interface { + Name() string + ToStruct() *TConfig +} + +// IStep represents interface for all types for teststeps, includes: +// StepRequest, StepRequestWithOptionalArgs, StepRequestValidation, StepRequestExtraction, +// StepTestCaseWithOptionalArgs, +// StepTransaction, StepRendezvous. type IStep interface { Name() string Type() string ToStruct() *TStep } -// ITestCase represents interface for all types for testcases. +// ITestCase represents interface for testcases, +// includes TestCase and TestCasePath. type ITestCase interface { ToTestCase() (*TestCase, error) ToTCase() (*TCase, error) } -// TestCase is a container for one testcase. -// used for testcase runner +// TestCase is a container for one testcase, which is used for testcase runner. +// TestCase implements ITestCase interface. type TestCase struct { - Config *TConfig + Config IConfig TestSteps []IStep } @@ -89,15 +129,23 @@ func (tc *TestCase) ToTestCase() (*TestCase, error) { return tc, nil } -type TestCasePath struct { - Path string +func (tc *TestCase) ToTCase() (*TCase, error) { + tCase := TCase{ + Config: tc.Config.ToStruct(), + } + for _, step := range tc.TestSteps { + tCase.TestSteps = append(tCase.TestSteps, step.ToStruct()) + } + return &tCase, nil } type testCaseSummary struct{} type stepData struct { - name string // step name - success bool // step execution result - responseLength int64 // response body length - exportVars map[string]interface{} // extract variables + name string // step name + stepType stepType // step type, testcase/request/transaction/rendezvous + success bool // step execution result + elapsed int64 // step execution time in millisecond(ms) + contentSize int64 // response body length + exportVars map[string]interface{} // extract variables } diff --git a/parser.go b/parser.go index 9bc0df8a..e7398cd7 100644 --- a/parser.go +++ b/parser.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/maja42/goval" + "github.com/rs/zerolog/log" "github.com/httprunner/hrp/internal/builtin" ) diff --git a/response.go b/response.go index 1fa9857b..f81de3f1 100644 --- a/response.go +++ b/response.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/jmespath/go-jmespath" + "github.com/rs/zerolog/log" "github.com/httprunner/hrp/internal/builtin" ) diff --git a/runner.go b/runner.go index 0f650f06..9f89495a 100644 --- a/runner.go +++ b/runner.go @@ -16,6 +16,7 @@ import ( "github.com/jinzhu/copier" "github.com/pkg/errors" + "github.com/rs/zerolog/log" "github.com/httprunner/hrp/internal/ga" ) @@ -32,8 +33,9 @@ func NewRunner(t *testing.T) *hrpRunner { t = &testing.T{} } return &hrpRunner{ - t: t, - debug: false, // default to turn off debug + t: t, + failfast: true, // default to failfast + debug: false, // default to turn off debug client: &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -41,14 +43,36 @@ func NewRunner(t *testing.T) *hrpRunner { Timeout: 30 * time.Second, }, sessionVariables: make(map[string]interface{}), + transactions: make(map[string]map[TransactionType]time.Time), } } type hrpRunner struct { t *testing.T + failfast bool debug bool client *http.Client sessionVariables map[string]interface{} + // transactions stores transaction timing info. + // key is transaction name, value is map of transaction type and time, e.g. start time and end time. + transactions map[string]map[TransactionType]time.Time + startTime time.Time // record start time of the testcase +} + +// Reset clears runner session variables. +func (r *hrpRunner) Reset() *hrpRunner { + log.Info().Msg("[init] Reset session variables") + r.sessionVariables = make(map[string]interface{}) + r.transactions = make(map[string]map[TransactionType]time.Time) + r.startTime = time.Now() + return r +} + +// SetFailfast configures whether to stop running when one step fails. +func (r *hrpRunner) SetFailfast(failfast bool) *hrpRunner { + log.Info().Bool("failfast", failfast).Msg("[init] SetFailfast") + r.failfast = failfast + return r } // SetDebug configures whether to log HTTP request and response content. @@ -84,6 +108,7 @@ func (r *hrpRunner) Run(testcases ...ITestCase) error { // report execution timing event defer ga.SendEvent(event.StartTiming("execution")) + r.Reset() for _, iTestCase := range testcases { testcase, err := iTestCase.ToTestCase() if err != nil { @@ -104,45 +129,60 @@ func (r *hrpRunner) runCase(testcase *TestCase) error { return err } - log.Info().Str("testcase", config.Name).Msg("run testcase start") - + log.Info().Str("testcase", config.Name()).Msg("run testcase start") + r.startTime = time.Now() for _, step := range testcase.TestSteps { _, err := r.runStep(step, config) if err != nil { - return err + if r.failfast { + log.Error().Err(err).Msg("abort running due to failfast setting") + return err + } + log.Warn().Err(err).Msg("run step failed, continue next step") } } - log.Info().Str("testcase", config.Name).Msg("run testcase end") + log.Info().Str("testcase", config.Name()).Msg("run testcase end") return nil } -func (r *hrpRunner) runStep(step IStep, config *TConfig) (stepResult *stepData, err error) { +func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, err error) { + // step type priority order: transaction > rendezvous > testcase > request + if stepTran, ok := step.(*StepTransaction); ok { + // transaction step + return r.runStepTransaction(stepTran.step.Transaction) + } else if stepRend, ok := step.(*StepRendezvous); ok { + // rendezvous step + return r.runStepRendezvous(stepRend.step.Rendezvous) + } + log.Info().Str("step", step.Name()).Msg("run step start") // copy step to avoid data racing copiedStep := &TStep{} if err = copier.Copy(copiedStep, step.ToStruct()); err != nil { log.Error().Err(err).Msg("copy step data failed") - return + return nil, err } + cfg := config.ToStruct() stepVariables := copiedStep.Variables // override variables // step variables > session variables (extracted variables from previous steps) stepVariables = mergeVariables(stepVariables, r.sessionVariables) // step variables > testcase config variables - stepVariables = mergeVariables(stepVariables, config.Variables) + stepVariables = mergeVariables(stepVariables, cfg.Variables) // parse step variables parsedVariables, err := parseVariables(stepVariables) if err != nil { - log.Error().Interface("variables", config.Variables).Err(err).Msg("parse step variables failed") - return + log.Error().Interface("variables", cfg.Variables).Err(err).Msg("parse step variables failed") + return nil, err } copiedStep.Variables = parsedVariables // avoid data racing - if _, ok := step.(*testcaseWithOptionalArgs); ok { + // step type priority order: testcase > request + if _, ok := step.(*StepTestCaseWithOptionalArgs); ok { // run referenced testcase log.Info().Str("testcase", copiedStep.Name).Msg("run referenced testcase") // TODO: override testcase config @@ -153,7 +193,7 @@ func (r *hrpRunner) runStep(step IStep, config *TConfig) (stepResult *stepData, } } else { // run request - copiedStep.Request.URL = buildURL(config.BaseURL, copiedStep.Request.URL) // avoid data racing + copiedStep.Request.URL = buildURL(cfg.BaseURL, copiedStep.Request.URL) // avoid data racing stepResult, err = r.runStepRequest(copiedStep) if err != nil { log.Error().Err(err).Msg("run request step failed") @@ -171,14 +211,72 @@ func (r *hrpRunner) runStep(step IStep, config *TConfig) (stepResult *stepData, Bool("success", stepResult.success). Interface("exportVars", stepResult.exportVars). Msg("run step end") - return + return stepResult, nil +} + +func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *stepData, err error) { + log.Info(). + Str("name", transaction.Name). + Str("type", string(transaction.Type)). + Msg("transaction") + + stepResult = &stepData{ + name: transaction.Name, + stepType: stepTypeTransaction, + success: true, + elapsed: 0, + contentSize: 0, // TODO: record transaction total response length + } + + // create transaction if not exists + if _, ok := r.transactions[transaction.Name]; !ok { + r.transactions[transaction.Name] = make(map[TransactionType]time.Time) + } + + // record transaction start time, override if already exists + if transaction.Type == TransactionStart { + r.transactions[transaction.Name][TransactionStart] = time.Now() + } + // record transaction end time, override if already exists + if transaction.Type == TransactionEnd { + r.transactions[transaction.Name][TransactionEnd] = time.Now() + + // if transaction start time not exists, use testcase start time instead + if _, ok := r.transactions[transaction.Name][TransactionStart]; !ok { + r.transactions[transaction.Name][TransactionStart] = r.startTime + } + + // calculate transaction duration + duration := r.transactions[transaction.Name][TransactionEnd].Sub( + r.transactions[transaction.Name][TransactionStart]) + stepResult.elapsed = duration.Milliseconds() + log.Info().Str("name", transaction.Name).Dur("elapsed", duration).Msg("transaction") + } + + return stepResult, nil +} + +func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, err error) { + log.Info(). + Str("name", rend.Name). + Float32("percent", rend.Percent). + Int64("number", rend.Number). + Int64("timeout", rend.Timeout). + Msg("rendezvous") + stepResult = &stepData{ + name: rend.Name, + stepType: stepTypeRendezvous, + success: true, + } + return stepResult, nil } func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) { stepResult = &stepData{ - name: step.Name, - success: false, - responseLength: 0, + name: step.Name, + stepType: stepTypeRequest, + success: false, + contentSize: 0, } rawUrl := step.Request.URL @@ -296,7 +394,9 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error } // do request action + start := time.Now() resp, err := r.client.Do(req) + stepResult.elapsed = time.Since(start).Milliseconds() if err != nil { return nil, errors.Wrap(err, "do request failed") } @@ -335,42 +435,50 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error } stepResult.success = true - stepResult.responseLength = resp.ContentLength - return + stepResult.contentSize = resp.ContentLength + return stepResult, nil } func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err error) { stepResult = &stepData{ - name: step.Name, - success: false, + name: step.Name, + stepType: stepTypeTestCase, + success: false, } testcase := step.TestCase + start := time.Now() err = r.runCase(testcase) - return + stepResult.elapsed = time.Since(start).Milliseconds() + if err != nil { + return stepResult, err + } + stepResult.success = true + return stepResult, nil } -func (r *hrpRunner) parseConfig(config *TConfig) error { +func (r *hrpRunner) parseConfig(config IConfig) error { + cfg := config.ToStruct() // parse config variables - parsedVariables, err := parseVariables(config.Variables) + parsedVariables, err := parseVariables(cfg.Variables) if err != nil { - log.Error().Interface("variables", config.Variables).Err(err).Msg("parse config variables failed") + log.Error().Interface("variables", cfg.Variables).Err(err).Msg("parse config variables failed") return err } - config.Variables = parsedVariables + cfg.Variables = parsedVariables // parse config name - parsedName, err := parseString(config.Name, config.Variables) + parsedName, err := parseString(cfg.Name, cfg.Variables) if err != nil { return err } - config.Name = convertString(parsedName) + cfg.Name = convertString(parsedName) // parse config base url - parsedBaseURL, err := parseString(config.BaseURL, config.Variables) + parsedBaseURL, err := parseString(cfg.BaseURL, cfg.Variables) if err != nil { return err } - config.BaseURL = convertString(parsedBaseURL) + cfg.BaseURL = convertString(parsedBaseURL) return nil } diff --git a/step.go b/step.go index 909e08ed..4bfcc167 100644 --- a/step.go +++ b/step.go @@ -3,277 +3,362 @@ package hrp import "fmt" // NewConfig returns a new constructed testcase config with specified testcase name. -func NewConfig(name string) *TConfig { - return &TConfig{ - Name: name, - Variables: make(map[string]interface{}), +func NewConfig(name string) *Config { + return &Config{ + cfg: &TConfig{ + Name: name, + Variables: make(map[string]interface{}), + }, } } +// Config implements IConfig interface. +type Config struct { + cfg *TConfig +} + // WithVariables sets variables for current testcase. -func (c *TConfig) WithVariables(variables map[string]interface{}) *TConfig { - c.Variables = variables +func (c *Config) WithVariables(variables map[string]interface{}) *Config { + c.cfg.Variables = variables return c } // SetBaseURL sets base URL for current testcase. -func (c *TConfig) SetBaseURL(baseURL string) *TConfig { - c.BaseURL = baseURL +func (c *Config) SetBaseURL(baseURL string) *Config { + c.cfg.BaseURL = baseURL return c } // SetVerifySSL sets whether to verify SSL for current testcase. -func (c *TConfig) SetVerifySSL(verify bool) *TConfig { - c.Verify = verify +func (c *Config) SetVerifySSL(verify bool) *Config { + c.cfg.Verify = verify return c } // WithParameters sets parameters for current testcase. -func (c *TConfig) WithParameters(parameters map[string]interface{}) *TConfig { - c.Parameters = parameters +func (c *Config) WithParameters(parameters map[string]interface{}) *Config { + c.cfg.Parameters = parameters return c } // ExportVars specifies variable names to export for current testcase. -func (c *TConfig) ExportVars(vars ...string) *TConfig { - c.Export = vars +func (c *Config) ExportVars(vars ...string) *Config { + c.cfg.Export = vars return c } // SetWeight sets weight for current testcase, which is used in load testing. -func (c *TConfig) SetWeight(weight int) *TConfig { - c.Weight = weight +func (c *Config) SetWeight(weight int) *Config { + c.cfg.Weight = weight return c } +// Name returns config name, this implements IConfig interface. +func (c *Config) Name() string { + return c.cfg.Name +} + +// ToStruct returns *TConfig, this implements IConfig interface. +func (c *Config) ToStruct() *TConfig { + return c.cfg +} + // NewStep returns a new constructed teststep with specified step name. -func NewStep(name string) *TStep { - return &TStep{ - Name: name, - Variables: make(map[string]interface{}), +func NewStep(name string) *StepRequest { + return &StepRequest{ + step: &TStep{ + Name: name, + Variables: make(map[string]interface{}), + }, } } +type StepRequest struct { + step *TStep +} + // WithVariables sets variables for current teststep. -func (s *TStep) WithVariables(variables map[string]interface{}) *TStep { - s.Variables = variables +func (s *StepRequest) WithVariables(variables map[string]interface{}) *StepRequest { + s.step.Variables = variables return s } // SetupHook adds a setup hook for current teststep. -func (s *TStep) SetupHook(hook string) *TStep { - s.SetupHooks = append(s.SetupHooks, hook) +func (s *StepRequest) SetupHook(hook string) *StepRequest { + s.step.SetupHooks = append(s.step.SetupHooks, hook) return s } // GET makes a HTTP GET request. -func (s *TStep) GET(url string) *requestWithOptionalArgs { - s.Request = &Request{ +func (s *StepRequest) GET(url string) *StepRequestWithOptionalArgs { + s.step.Request = &Request{ Method: httpGET, URL: url, } - return &requestWithOptionalArgs{ - step: s, + return &StepRequestWithOptionalArgs{ + step: s.step, } } // HEAD makes a HTTP HEAD request. -func (s *TStep) HEAD(url string) *requestWithOptionalArgs { - s.Request = &Request{ +func (s *StepRequest) HEAD(url string) *StepRequestWithOptionalArgs { + s.step.Request = &Request{ Method: httpHEAD, URL: url, } - return &requestWithOptionalArgs{ - step: s, + return &StepRequestWithOptionalArgs{ + step: s.step, } } // POST makes a HTTP POST request. -func (s *TStep) POST(url string) *requestWithOptionalArgs { - s.Request = &Request{ +func (s *StepRequest) POST(url string) *StepRequestWithOptionalArgs { + s.step.Request = &Request{ Method: httpPOST, URL: url, } - return &requestWithOptionalArgs{ - step: s, + return &StepRequestWithOptionalArgs{ + step: s.step, } } // PUT makes a HTTP PUT request. -func (s *TStep) PUT(url string) *requestWithOptionalArgs { - s.Request = &Request{ +func (s *StepRequest) PUT(url string) *StepRequestWithOptionalArgs { + s.step.Request = &Request{ Method: httpPUT, URL: url, } - return &requestWithOptionalArgs{ - step: s, + return &StepRequestWithOptionalArgs{ + step: s.step, } } // DELETE makes a HTTP DELETE request. -func (s *TStep) DELETE(url string) *requestWithOptionalArgs { - s.Request = &Request{ +func (s *StepRequest) DELETE(url string) *StepRequestWithOptionalArgs { + s.step.Request = &Request{ Method: httpDELETE, URL: url, } - return &requestWithOptionalArgs{ - step: s, + return &StepRequestWithOptionalArgs{ + step: s.step, } } // OPTIONS makes a HTTP OPTIONS request. -func (s *TStep) OPTIONS(url string) *requestWithOptionalArgs { - s.Request = &Request{ +func (s *StepRequest) OPTIONS(url string) *StepRequestWithOptionalArgs { + s.step.Request = &Request{ Method: httpOPTIONS, URL: url, } - return &requestWithOptionalArgs{ - step: s, + return &StepRequestWithOptionalArgs{ + step: s.step, } } // PATCH makes a HTTP PATCH request. -func (s *TStep) PATCH(url string) *requestWithOptionalArgs { - s.Request = &Request{ +func (s *StepRequest) PATCH(url string) *StepRequestWithOptionalArgs { + s.step.Request = &Request{ Method: httpPATCH, URL: url, } - return &requestWithOptionalArgs{ - step: s, + return &StepRequestWithOptionalArgs{ + step: s.step, } } // CallRefCase calls a referenced testcase. -func (s *TStep) CallRefCase(tc *TestCase) *testcaseWithOptionalArgs { - s.TestCase = tc - return &testcaseWithOptionalArgs{ - step: s, +func (s *StepRequest) CallRefCase(tc *TestCase) *StepTestCaseWithOptionalArgs { + s.step.TestCase = tc + return &StepTestCaseWithOptionalArgs{ + step: s.step, } } -// implements IStep interface -type requestWithOptionalArgs struct { +// StartTransaction starts a transaction. +func (s *StepRequest) StartTransaction(name string) *StepTransaction { + s.step.Transaction = &Transaction{ + Name: name, + Type: TransactionStart, + } + return &StepTransaction{ + step: s.step, + } +} + +// EndTransaction ends a transaction. +func (s *StepRequest) EndTransaction(name string) *StepTransaction { + s.step.Transaction = &Transaction{ + Name: name, + Type: TransactionEnd, + } + return &StepTransaction{ + step: s.step, + } +} + +// StepRequestWithOptionalArgs implements IStep interface. +type StepRequestWithOptionalArgs struct { step *TStep } // SetVerify sets whether to verify SSL for current HTTP request. -func (s *requestWithOptionalArgs) SetVerify(verify bool) *requestWithOptionalArgs { +func (s *StepRequestWithOptionalArgs) SetVerify(verify bool) *StepRequestWithOptionalArgs { s.step.Request.Verify = verify return s } // SetTimeout sets timeout for current HTTP request. -func (s *requestWithOptionalArgs) SetTimeout(timeout float32) *requestWithOptionalArgs { +func (s *StepRequestWithOptionalArgs) SetTimeout(timeout float32) *StepRequestWithOptionalArgs { s.step.Request.Timeout = timeout return s } // SetProxies sets proxies for current HTTP request. -func (s *requestWithOptionalArgs) SetProxies(proxies map[string]string) *requestWithOptionalArgs { +func (s *StepRequestWithOptionalArgs) SetProxies(proxies map[string]string) *StepRequestWithOptionalArgs { // TODO return s } // SetAllowRedirects sets whether to allow redirects for current HTTP request. -func (s *requestWithOptionalArgs) SetAllowRedirects(allowRedirects bool) *requestWithOptionalArgs { +func (s *StepRequestWithOptionalArgs) SetAllowRedirects(allowRedirects bool) *StepRequestWithOptionalArgs { s.step.Request.AllowRedirects = allowRedirects return s } // SetAuth sets auth for current HTTP request. -func (s *requestWithOptionalArgs) SetAuth(auth map[string]string) *requestWithOptionalArgs { +func (s *StepRequestWithOptionalArgs) SetAuth(auth map[string]string) *StepRequestWithOptionalArgs { // TODO return s } // WithParams sets HTTP request params for current step. -func (s *requestWithOptionalArgs) WithParams(params map[string]interface{}) *requestWithOptionalArgs { +func (s *StepRequestWithOptionalArgs) WithParams(params map[string]interface{}) *StepRequestWithOptionalArgs { s.step.Request.Params = params return s } // WithHeaders sets HTTP request headers for current step. -func (s *requestWithOptionalArgs) WithHeaders(headers map[string]string) *requestWithOptionalArgs { +func (s *StepRequestWithOptionalArgs) WithHeaders(headers map[string]string) *StepRequestWithOptionalArgs { s.step.Request.Headers = headers return s } // WithCookies sets HTTP request cookies for current step. -func (s *requestWithOptionalArgs) WithCookies(cookies map[string]string) *requestWithOptionalArgs { +func (s *StepRequestWithOptionalArgs) WithCookies(cookies map[string]string) *StepRequestWithOptionalArgs { s.step.Request.Cookies = cookies return s } // WithBody sets HTTP request body for current step. -func (s *requestWithOptionalArgs) WithBody(body interface{}) *requestWithOptionalArgs { +func (s *StepRequestWithOptionalArgs) WithBody(body interface{}) *StepRequestWithOptionalArgs { s.step.Request.Body = body return s } // TeardownHook adds a teardown hook for current teststep. -func (s *requestWithOptionalArgs) TeardownHook(hook string) *requestWithOptionalArgs { +func (s *StepRequestWithOptionalArgs) TeardownHook(hook string) *StepRequestWithOptionalArgs { s.step.TeardownHooks = append(s.step.TeardownHooks, hook) return s } // Validate switches to step validation. -func (s *requestWithOptionalArgs) Validate() *stepRequestValidation { - return &stepRequestValidation{ +func (s *StepRequestWithOptionalArgs) Validate() *StepRequestValidation { + return &StepRequestValidation{ step: s.step, } } // Extract switches to step extraction. -func (s *requestWithOptionalArgs) Extract() *stepRequestExtraction { +func (s *StepRequestWithOptionalArgs) Extract() *StepRequestExtraction { s.step.Extract = make(map[string]string) - return &stepRequestExtraction{ + return &StepRequestExtraction{ step: s.step, } } -func (s *requestWithOptionalArgs) Name() string { +func (s *StepRequestWithOptionalArgs) Name() string { if s.step.Name != "" { return s.step.Name } return fmt.Sprintf("%s %s", s.step.Request.Method, s.step.Request.URL) } -func (s *requestWithOptionalArgs) Type() string { +func (s *StepRequestWithOptionalArgs) Type() string { return fmt.Sprintf("request-%v", s.step.Request.Method) } -func (s *requestWithOptionalArgs) ToStruct() *TStep { +func (s *StepRequestWithOptionalArgs) ToStruct() *TStep { return s.step } -// implements IStep interface -type testcaseWithOptionalArgs struct { +// StepTestCaseWithOptionalArgs implements IStep interface. +type StepTestCaseWithOptionalArgs struct { step *TStep } // TeardownHook adds a teardown hook for current teststep. -func (s *testcaseWithOptionalArgs) TeardownHook(hook string) *testcaseWithOptionalArgs { +func (s *StepTestCaseWithOptionalArgs) TeardownHook(hook string) *StepTestCaseWithOptionalArgs { s.step.TeardownHooks = append(s.step.TeardownHooks, hook) return s } // Export specifies variable names to export from referenced testcase for current step. -func (s *testcaseWithOptionalArgs) Export(names ...string) *testcaseWithOptionalArgs { +func (s *StepTestCaseWithOptionalArgs) Export(names ...string) *StepTestCaseWithOptionalArgs { s.step.Export = append(s.step.Export, names...) return s } -func (s *testcaseWithOptionalArgs) Name() string { +func (s *StepTestCaseWithOptionalArgs) Name() string { if s.step.Name != "" { return s.step.Name } - return s.step.TestCase.Config.Name + return s.step.TestCase.Config.Name() } -func (s *testcaseWithOptionalArgs) Type() string { +func (s *StepTestCaseWithOptionalArgs) Type() string { return "testcase" } -func (s *testcaseWithOptionalArgs) ToStruct() *TStep { +func (s *StepTestCaseWithOptionalArgs) ToStruct() *TStep { + return s.step +} + +// StepTransaction implements IStep interface. +type StepTransaction struct { + step *TStep +} + +func (s *StepTransaction) Name() string { + if s.step.Name != "" { + return s.step.Name + } + return fmt.Sprintf("transaction %s %s", s.step.Transaction.Name, s.step.Transaction.Type) +} + +func (s *StepTransaction) Type() string { + return "transaction" +} + +func (s *StepTransaction) ToStruct() *TStep { + return s.step +} + +// StepRendezvous implements IStep interface. +type StepRendezvous struct { + step *TStep +} + +func (s *StepRendezvous) Name() string { + if s.step.Name != "" { + return s.step.Name + } + return s.step.Rendezvous.Name +} + +func (s *StepRendezvous) Type() string { + return "rendezvous" +} + +func (s *StepRendezvous) ToStruct() *TStep { return s.step } diff --git a/validate.go b/validate.go index e522ac9c..f5d67a8a 100644 --- a/validate.go +++ b/validate.go @@ -4,27 +4,27 @@ import ( "fmt" ) -// implements IStep interface -type stepRequestValidation struct { +// StepRequestValidation implements IStep interface. +type StepRequestValidation struct { step *TStep } -func (s *stepRequestValidation) Name() string { +func (s *StepRequestValidation) Name() string { if s.step.Name != "" { return s.step.Name } return fmt.Sprintf("%s %s", s.step.Request.Method, s.step.Request.URL) } -func (s *stepRequestValidation) Type() string { +func (s *StepRequestValidation) Type() string { return fmt.Sprintf("request-%v", s.step.Request.Method) } -func (s *stepRequestValidation) ToStruct() *TStep { +func (s *StepRequestValidation) ToStruct() *TStep { return s.step } -func (s *stepRequestValidation) AssertEqual(jmesPath string, expected interface{}, msg string) *stepRequestValidation { +func (s *StepRequestValidation) AssertEqual(jmesPath string, expected interface{}, msg string) *StepRequestValidation { v := Validator{ Check: jmesPath, Assert: "equals", @@ -35,7 +35,7 @@ func (s *stepRequestValidation) AssertEqual(jmesPath string, expected interface{ return s } -func (s *stepRequestValidation) AssertStartsWith(jmesPath string, expected interface{}, msg string) *stepRequestValidation { +func (s *StepRequestValidation) AssertStartsWith(jmesPath string, expected interface{}, msg string) *StepRequestValidation { v := Validator{ Check: jmesPath, Assert: "startswith", @@ -46,7 +46,7 @@ func (s *stepRequestValidation) AssertStartsWith(jmesPath string, expected inter return s } -func (s *stepRequestValidation) AssertEndsWith(jmesPath string, expected interface{}, msg string) *stepRequestValidation { +func (s *StepRequestValidation) AssertEndsWith(jmesPath string, expected interface{}, msg string) *StepRequestValidation { v := Validator{ Check: jmesPath, Assert: "endswith", @@ -57,7 +57,7 @@ func (s *stepRequestValidation) AssertEndsWith(jmesPath string, expected interfa return s } -func (s *stepRequestValidation) AssertLengthEqual(jmesPath string, expected interface{}, msg string) *stepRequestValidation { +func (s *StepRequestValidation) AssertLengthEqual(jmesPath string, expected interface{}, msg string) *StepRequestValidation { v := Validator{ Check: jmesPath, Assert: "length_equals",