From fe2c2f170e39785b89e28e1e7e5e637dcb0c651c Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 16 Apr 2022 00:35:25 +0800 Subject: [PATCH 1/7] refactor: ParametersIterator --- hrp/boomer.go | 15 +- hrp/config.go | 59 ------- hrp/parameters.go | 345 +++++++++++++++++++++++++++++++++++++++++ hrp/parameters_test.go | 299 +++++++++++++++++++++++++++++++++++ hrp/parser.go | 177 --------------------- hrp/parser_test.go | 150 ------------------ hrp/runner.go | 12 +- hrp/session.go | 26 ++-- 8 files changed, 670 insertions(+), 413 deletions(-) create mode 100644 hrp/parameters.go create mode 100644 hrp/parameters_test.go diff --git a/hrp/boomer.go b/hrp/boomer.go index 8a59defb..7bba9285 100644 --- a/hrp/boomer.go +++ b/hrp/boomer.go @@ -83,7 +83,6 @@ func (b *HRPBoomer) convertBoomerTask(testcase *TestCase, rendezvousList []*Rend b.plugins = append(b.plugins, sessionRunner.parser.plugin) b.pluginsMutex.Unlock() } - sessionRunner.resetSession() // broadcast to all rendezvous at once when spawn done go func() { @@ -93,6 +92,10 @@ func (b *HRPBoomer) convertBoomerTask(testcase *TestCase, rendezvousList []*Rend } }() + // set paramters mode for load testing + parametersIterator := sessionRunner.parametersIterator + parametersIterator.SetUnlimitedMode() + return &boomer.Task{ Name: testcase.Config.Name, Weight: testcase.Config.Weight, @@ -100,15 +103,11 @@ func (b *HRPBoomer) convertBoomerTask(testcase *TestCase, rendezvousList []*Rend testcaseSuccess := true // flag whole testcase result transactionSuccess := true // flag current transaction result - var parameterVariables map[string]interface{} - // iterate through all parameter iterators and update case variables - for _, it := range sessionRunner.parsedConfig.ParametersSetting.Iterators { - if it.HasNext() { - parameterVariables = it.Next() - } + if parametersIterator.HasNext() { + sessionRunner.updateConfigVariables(parametersIterator.Next()) } - sessionRunner.updateConfigVariables(parameterVariables) + sessionRunner.resetSession() startTime := time.Now() for _, step := range testcase.TestSteps { stepResult, err := step.Run(sessionRunner) diff --git a/hrp/config.go b/hrp/config.go index 7a3683fd..574ba2d8 100644 --- a/hrp/config.go +++ b/hrp/config.go @@ -1,10 +1,7 @@ package hrp import ( - "math/rand" "reflect" - "sync" - "time" "github.com/httprunner/httprunner/hrp/internal/builtin" ) @@ -158,59 +155,3 @@ const ( var ( thinkTimeDefaultRandom = map[string]float64{"min_percentage": 0.5, "max_percentage": 1.5} ) - -type TParamsConfig struct { - Strategy interface{} `json:"strategy,omitempty" yaml:"strategy,omitempty"` // map[string]string、string - Iteration int `json:"iteration,omitempty" yaml:"iteration,omitempty"` - Iterators []*Iterator `json:"parameterIterator,omitempty" yaml:"parameterIterator,omitempty"` // 保存参数的迭代器 -} - -type Iterator struct { - sync.Mutex - data iteratorParamsType - strategy iteratorStrategyType // random, sequential - iteration int - index int -} - -type iteratorStrategyType string - -const ( - strategyRandom iteratorStrategyType = "random" - strategySequential iteratorStrategyType = "sequential" -) - -type iteratorParamsType []map[string]interface{} - -func (params iteratorParamsType) Iterator() *Iterator { - return &Iterator{ - data: params, - iteration: len(params), - index: 0, - } -} - -func (iter *Iterator) HasNext() bool { - if iter.iteration == -1 { - return true - } - return iter.index < iter.iteration -} - -func (iter *Iterator) Next() (value map[string]interface{}) { - iter.Lock() - defer iter.Unlock() - if len(iter.data) == 0 { - iter.index++ - return map[string]interface{}{} - } - if iter.strategy == strategyRandom { - randSource := rand.New(rand.NewSource(time.Now().Unix())) - randIndex := randSource.Intn(len(iter.data)) - value = iter.data[randIndex] - } else { - value = iter.data[iter.index%len(iter.data)] - } - iter.index++ - return value -} diff --git a/hrp/parameters.go b/hrp/parameters.go new file mode 100644 index 00000000..a42f87eb --- /dev/null +++ b/hrp/parameters.go @@ -0,0 +1,345 @@ +package hrp + +import ( + "math/rand" + "reflect" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +type TParamsConfig struct { + Strategy iteratorStrategy `json:"strategy,omitempty" yaml:"strategy,omitempty"` // overall strategy + Strategies map[string]iteratorStrategy `json:"strategies,omitempty" yaml:"strategies,omitempty"` // map[string]string、string + Limit int `json:"limit,omitempty" yaml:"limit,omitempty"` +} + +type iteratorStrategy string + +const ( + strategySequential iteratorStrategy = "sequential" + strategyRandom iteratorStrategy = "random" + strategyUnique iteratorStrategy = "unique" +) + +/* +[ + {"username": "test1", "password": "111111"}, + {"username": "test2", "password": "222222"}, +] +*/ +type Parameters []map[string]interface{} + +func initParametersIterator(cfg *TConfig) (*ParametersIterator, error) { + parameters, err := loadParameters(cfg.Parameters, cfg.Variables) + if err != nil { + return nil, err + } + return newParametersIterator(parameters, cfg.ParametersSetting), nil +} + +func newParametersIterator(parameters map[string]Parameters, config *TParamsConfig) *ParametersIterator { + iterator := &ParametersIterator{ + data: parameters, + hasNext: true, + sequentialParameters: nil, + randomParameterNames: nil, + limit: config.Limit, + index: 0, + } + + if len(parameters) == 0 { + iterator.hasNext = false + return iterator + } + + parametersList := make([]Parameters, 0) + for paramName := range parameters { + // check parameter individual strategy + strategy, ok := config.Strategies[paramName] + if !ok { + // default to overall strategy + strategy = config.Strategy + } + + // group parameters by strategy + if strategy == strategyRandom { + iterator.randomParameterNames = append(iterator.randomParameterNames, paramName) + } else { + parametersList = append(parametersList, parameters[paramName]) + } + } + + // generate cartesian product for sequential parameters + iterator.sequentialParameters = genCartesianProduct(parametersList) + if iterator.limit == 0 { + if len(iterator.sequentialParameters) > 0 { + iterator.limit = len(iterator.sequentialParameters) + } else { + iterator.limit = 1 + } + } + + return iterator +} + +type ParametersIterator struct { + sync.Mutex + data map[string]Parameters + hasNext bool // cache query result + sequentialParameters Parameters // cartesian product for sequential parameters + randomParameterNames []string // value is parameter names + limit int + index int +} + +// SetUnlimitedMode is used for load testing +func (iter *ParametersIterator) SetUnlimitedMode() { + iter.limit = -1 +} + +func (iter *ParametersIterator) HasNext() bool { + if !iter.hasNext { + return false + } + + // unlimited mode + if iter.limit == -1 { + return true + } + + // reached limit + if iter.index >= iter.limit { + // cache query result + iter.hasNext = false + return false + } + + return true +} + +func (iter *ParametersIterator) Next() map[string]interface{} { + iter.Lock() + defer iter.Unlock() + + if !iter.hasNext { + return nil + } + + if len(iter.data) == 0 { + iter.hasNext = false + return nil + } + + selectedParameters := make(map[string]interface{}) + if iter.index < len(iter.sequentialParameters) { + selectedParameters = iter.sequentialParameters[iter.index] + } + + for _, paramName := range iter.randomParameterNames { + randSource := rand.New(rand.NewSource(time.Now().Unix())) + randIndex := randSource.Intn(len(iter.data[paramName])) + selectedParameters[paramName] = iter.data[paramName][randIndex] + } + + iter.index++ + if iter.limit > 0 && iter.index >= iter.limit { + iter.hasNext = false + } + + return selectedParameters +} + +func genCartesianProduct(multiParameters []Parameters) Parameters { + if len(multiParameters) == 0 { + return nil + } + + cartesianProduct := multiParameters[0] + for i := 0; i < len(multiParameters)-1; i++ { + var tempProduct Parameters + for _, param1 := range cartesianProduct { + for _, param2 := range multiParameters[i+1] { + tempProduct = append(tempProduct, mergeVariables(param1, param2)) + } + } + cartesianProduct = tempProduct + } + + return cartesianProduct +} + +/* loadParameters loads parameters from multiple sources. + +parameter value may be in three types: + (1) data list, e.g. ["iOS/10.1", "iOS/10.2", "iOS/10.3"] + (2) call built-in parameterize function, "${parameterize(account.csv)}" + (3) call custom function in debugtalk.py, "${gen_app_version()}" + +configParameters = { + "user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"], // case 1 + "username-password": "${parameterize(account.csv)}", // case 2 + "app_version": "${gen_app_version()}", // case 3 +} + +=> + +{ + "user_agent": [ + {"user_agent": "iOS/10.1"}, + {"user_agent": "iOS/10.2"}, + {"user_agent": "iOS/10.3"}, + ], + "username-password": [ + {"username": "test1", "password": "111111"}, + {"username": "test2", "password": "222222"}, + ], + "app_version": [ + {"app_version": "1.0.0"}, + {"app_version": "1.0.1"}, + ] +} +*/ +func loadParameters(configParameters map[string]interface{}, variablesMapping map[string]interface{}) ( + map[string]Parameters, error) { + + if len(configParameters) == 0 { + return nil, nil + } + + parsedParameters := make(map[string]Parameters) + + for k, v := range configParameters { + var parametersRawList interface{} + rawValue := reflect.ValueOf(v) + + switch rawValue.Kind() { + case reflect.Slice: + // case 1 + // e.g. user_agent: ["iOS/10.1", "iOS/10.2"] + // => ["iOS/10.1", "iOS/10.2"] + parametersRawList = rawValue.Interface() + + case reflect.String: + // case 2 or case 3 + // e.g. username-password: ${parameterize(examples/hrp/account.csv)} + // => [{"username": "test1", "password": "111111"}, {"username": "test2", "password": "222222"}] + // => [["test1", "111111"], ["test2", "222222"]] + // e.g. "app_version": "${gen_app_version()}" + // => ["1.0.0", "1.0.1"] + parsedParameterContent, err := newParser().ParseString(rawValue.String(), variablesMapping) + if err != nil { + log.Error().Err(err). + Str("parametersRawContent", rawValue.String()). + Msg("parse parameters content failed") + return nil, err + } + + parsedParameterRawValue := reflect.ValueOf(parsedParameterContent) + if parsedParameterRawValue.Kind() != reflect.Slice { + log.Error(). + Interface("parsedParameterContent", parsedParameterRawValue). + Msg("parsed parameters content is not slice") + return nil, errors.New("parsed parameters content should be slice") + } + parametersRawList = parsedParameterRawValue.Interface() + + default: + log.Error(). + Interface("parameters", configParameters). + Msg("config parameters raw value should be slice or string (functions call)") + return nil, errors.New("config parameters raw value format error") + } + + parameterSlice, err := convertParameters(k, parametersRawList) + if err != nil { + return nil, err + } + parsedParameters[k] = parameterSlice + } + return parsedParameters, nil +} + +/* convert parameters to standard format + +key and parametersRawList may be in three types: + +case 1: + key = "user_agent" + parametersRawList = ["iOS/10.1", "iOS/10.2"] + +case 2: + key = "username-password" + parametersRawList = [{"username": "test1", "password": "111111"}, {"username": "test2", "password": "222222"}] + +case 3: + key = "username-password" + parametersRawList = [["test1", "111111"], ["test2", "222222"]] +*/ +func convertParameters(key string, parametersRawList interface{}) (parameterSlice []map[string]interface{}, err error) { + parametersRawSlice := reflect.ValueOf(parametersRawList) + if parametersRawSlice.Kind() != reflect.Slice { + return nil, errors.New("parameters raw value is not list") + } + + // ["user_agent"], ["username", "password"], ["app_version"] + parameterNames := strings.Split(key, "-") + + for i := 0; i < parametersRawSlice.Len(); i++ { + parametersLine := make(map[string]interface{}) + elem := parametersRawSlice.Index(i) + switch elem.Kind() { + case reflect.Slice: + // case 3 + // e.g. "username-password": ["test1", "111111"] + // => {"username": "test1", "password": "111111"} + if len(parameterNames) != elem.Len() { + log.Error(). + Strs("parameterNames", parameterNames). + Int("lineIndex", i). + Interface("content", elem.Interface()). + Msg("parameters line length does not match to names length") + return nil, errors.New("parameters line length does not match to names length") + } + + for j := 0; j < elem.Len(); j++ { + parametersLine[parameterNames[j]] = elem.Index(j).Interface() + } + + case reflect.Map: + // case 2 + // e.g. "username-password": {"username": "test1", "password": "111111", "other": "111"} + // => {"username": "test1", "password": "passwd1"} + for _, name := range parameterNames { + lineMap := elem.Interface().(map[string]interface{}) + if _, ok := lineMap[name]; ok { + parametersLine[name] = elem.MapIndex(reflect.ValueOf(name)).Interface() + } else { + log.Error(). + Strs("parameterNames", parameterNames). + Str("name", name). + Msg("parameter name not found") + return nil, errors.New("parameter name not found") + } + } + + default: + // case 1 + // e.g. "user_agent": "iOS/10.1" + // -> {"user_agent": "iOS/10.1"} + if len(parameterNames) != 1 { + log.Error(). + Strs("parameterNames", parameterNames). + Int("lineIndex", i). + Msg("parameters format error") + return nil, errors.New("parameters format error") + } + parametersLine[parameterNames[0]] = elem.Interface() + } + parameterSlice = append(parameterSlice, parametersLine) + } + return parameterSlice, nil +} diff --git a/hrp/parameters_test.go b/hrp/parameters_test.go new file mode 100644 index 00000000..5f834313 --- /dev/null +++ b/hrp/parameters_test.go @@ -0,0 +1,299 @@ +package hrp + +import ( + "fmt" + "testing" + + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" +) + +func TestLoadParameters(t *testing.T) { + testData := []struct { + configParameters map[string]interface{} + loadedParameters map[string]Parameters + }{ + { + map[string]interface{}{ + "username-password": fmt.Sprintf("${parameterize(%s/$file)}", hrpExamplesDir), + }, + map[string]Parameters{ + "username-password": { + {"username": "test1", "password": "111111"}, + {"username": "test2", "password": "222222"}, + {"username": "test3", "password": "333333"}, + }, + }, + }, + { + map[string]interface{}{ + "username-password": [][]interface{}{ + {"test1", "111111"}, + {"test2", "222222"}, + }, + "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}, + "app_version": []interface{}{4.0}, + }, + map[string]Parameters{ + "username-password": { + {"username": "test1", "password": "111111"}, + {"username": "test2", "password": "222222"}, + }, + "user_agent": { + {"user_agent": "IOS/10.1"}, + {"user_agent": "IOS/10.2"}, + }, + "app_version": { + {"app_version": 4.0}, + }, + }, + }, + { + map[string]interface{}{}, + nil, + }, + { + nil, + nil, + }, + } + + variablesMapping := map[string]interface{}{ + "file": "account.csv", + } + for _, data := range testData { + value, err := loadParameters(data.configParameters, variablesMapping) + if !assert.Nil(t, err) { + t.Fatal() + } + if !assert.Equal(t, data.loadedParameters, value) { + t.Fatal() + } + } +} + +func TestLoadParametersError(t *testing.T) { + testData := []struct { + configParameters map[string]interface{} + }{ + { + map[string]interface{}{ + "username_password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), + "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}}, + }, + { + map[string]interface{}{ + "username-password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), + "user-agent": []interface{}{"IOS/10.1", "IOS/10.2"}}, + }, + { + map[string]interface{}{ + "username-password": fmt.Sprintf("${param(%s/account.csv)}", hrpExamplesDir), + "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}}, + }, + } + for _, data := range testData { + _, err := loadParameters(data.configParameters, map[string]interface{}{}) + if !assert.Error(t, err) { + t.Fatal() + } + } +} + +func TestInitParametersIterator(t *testing.T) { + configParameters := map[string]interface{}{ + "username-password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), // 3 + "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}, + "app_version": []interface{}{4.0}, + } + testData := []struct { + cfg *TConfig + expectLimit int + }{ + { + &TConfig{ + Parameters: configParameters, + ParametersSetting: &TParamsConfig{}, + }, + 6, + }, + { + &TConfig{ + Parameters: configParameters, + ParametersSetting: &TParamsConfig{ + Strategy: "random", + }, + }, + 1, + }, + { + &TConfig{ + Parameters: configParameters, + ParametersSetting: &TParamsConfig{ + Strategies: map[string]iteratorStrategy{ + "username-password": "random", + }, + }, + }, + 2, + }, + } + for _, data := range testData { + iterator, err := initParametersIterator(data.cfg) + if !assert.Nil(t, err) { + t.Fatal() + } + if !assert.Equal(t, data.expectLimit, iterator.limit) { + t.Fatal() + } + + for i := 0; i < data.expectLimit; i++ { + if !assert.True(t, iterator.HasNext()) { + t.Fatal() + } + log.Info().Interface("next", iterator.Next()).Msg("get next parameters") + } + // should not have next + if !assert.False(t, iterator.HasNext()) { + t.Fatal() + } + } +} + +func TestGenCartesianProduct(t *testing.T) { + testData := []struct { + multiParameters []Parameters + expect Parameters + }{ + { + []Parameters{ + { + {"app_version": 4.0}, + }, + { + {"username": "test1", "password": "111111"}, + {"username": "test2", "password": "222222"}, + }, + { + {"user_agent": "iOS/10.1"}, + {"user_agent": "iOS/10.2"}, + }, + }, + Parameters{ + {"app_version": 4.0, "password": "111111", "user_agent": "iOS/10.1", "username": "test1"}, + {"app_version": 4.0, "password": "111111", "user_agent": "iOS/10.2", "username": "test1"}, + {"app_version": 4.0, "password": "222222", "user_agent": "iOS/10.1", "username": "test2"}, + {"app_version": 4.0, "password": "222222", "user_agent": "iOS/10.2", "username": "test2"}, + }, + }, + { + nil, + nil, + }, + { + []Parameters{}, + nil, + }, + } + + for _, data := range testData { + parameters := genCartesianProduct(data.multiParameters) + if !assert.Equal(t, data.expect, parameters) { + t.Fatal() + } + } +} + +func TestConvertParameters(t *testing.T) { + testData := []struct { + key string + parametersRawList interface{} + expect []map[string]interface{} + }{ + { + "username-password", + []map[string]interface{}{ + {"username": "test1", "password": 111111, "other": "111"}, + {"username": "test2", "password": 222222, "other": "222"}, + }, + []map[string]interface{}{ + {"username": "test1", "password": 111111}, + {"username": "test2", "password": 222222}, + }, + }, + { + "username-password", + [][]string{ + {"test1", "111111"}, + {"test2", "222222"}, + }, + []map[string]interface{}{ + {"username": "test1", "password": "111111"}, + {"username": "test2", "password": "222222"}, + }, + }, + { + "app_version", + []float64{3.1, 3.0}, + []map[string]interface{}{ + {"app_version": 3.1}, + {"app_version": 3.0}, + }, + }, + { + "user_agent", + []string{"iOS/10.1", "iOS/10.2"}, + []map[string]interface{}{ + {"user_agent": "iOS/10.1"}, + {"user_agent": "iOS/10.2"}, + }, + }, + } + + for _, data := range testData { + value, err := convertParameters(data.key, data.parametersRawList) + if !assert.Nil(t, err) { + t.Fatal() + } + if !assert.Equal(t, data.expect, value) { + t.Fatal() + } + } +} + +func TestConvertParametersError(t *testing.T) { + testData := []struct { + key string + parametersRawList interface{} + }{ + { + "app_version", + 123, // not slice + }, + { + "app_version", + "123", // not slice + }, + { + "username-password", + []map[string]interface{}{ // parameter names not match + {"username": "test1", "other": "111"}, + {"username": "test2", "other": "222"}, + }, + }, + { + "username-password", + [][]string{ // parameter names length not match + {"test1"}, + {"test2"}, + }, + }, + } + + for _, data := range testData { + _, err := convertParameters(data.key, data.parametersRawList) + if !assert.Error(t, err) { + t.Fatal() + } + } +} diff --git a/hrp/parser.go b/hrp/parser.go index 46ab0f47..3a0d03bb 100644 --- a/hrp/parser.go +++ b/hrp/parser.go @@ -10,7 +10,6 @@ import ( "strings" "github.com/maja42/goval" - "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/httprunner/funplugin" @@ -532,179 +531,3 @@ func findallVariables(raw string) variableSet { return varSet } - -func genCartesianProduct(paramsMap map[string]iteratorParamsType) iteratorParamsType { - if len(paramsMap) == 0 { - return nil - } - var params []iteratorParamsType - for _, v := range paramsMap { - params = append(params, v) - } - var cartesianProduct iteratorParamsType - cartesianProduct = params[0] - for i := 0; i < len(params)-1; i++ { - var tempProduct iteratorParamsType - for _, param1 := range cartesianProduct { - for _, param2 := range params[i+1] { - tempProduct = append(tempProduct, mergeVariables(param1, param2)) - } - } - cartesianProduct = tempProduct - } - return cartesianProduct -} - -func parseParameters(parameters map[string]interface{}, variablesMapping map[string]interface{}) (map[string]iteratorParamsType, error) { - if len(parameters) == 0 { - return nil, nil - } - parsedParametersSlice := make(map[string]iteratorParamsType) - var err error - for k, v := range parameters { - var parameterSlice iteratorParamsType - rawValue := reflect.ValueOf(v) - switch rawValue.Kind() { - case reflect.String: - // e.g. username-password: ${parameterize(examples/hrp/account.csv)} -> [{"username": "test1", "password": "111111"}, {"username": "test2", "password": "222222"}] - var parsedParameterContent interface{} - parsedParameterContent, err = newParser().ParseString(rawValue.String(), variablesMapping) - if err != nil { - log.Error().Interface("parameterContent", rawValue).Msg("[parseParameters] parse parameter content error") - return nil, err - } - parsedParameterRawValue := reflect.ValueOf(parsedParameterContent) - if parsedParameterRawValue.Kind() != reflect.Slice { - log.Error().Interface("parameterContent", parsedParameterRawValue).Msg("[parseParameters] parsed parameter content should be slice") - return nil, errors.New("parsed parameter content should be slice") - } - parameterSlice, err = parseSlice(k, parsedParameterRawValue.Interface()) - case reflect.Slice: - // e.g. user_agent: ["iOS/10.1", "iOS/10.2"] -> [{"user_agent": "iOS/10.1"}, {"user_agent": "iOS/10.2"}] - parameterSlice, err = parseSlice(k, rawValue.Interface()) - default: - log.Error().Interface("parameter", parameters).Msg("[parseParameters] parameter content should be slice or text(functions call)") - return nil, errors.New("parameter content should be slice or text(functions call)") - } - if err != nil { - return nil, err - } - parsedParametersSlice[k] = parameterSlice - } - return parsedParametersSlice, nil -} - -func parseSlice(parameterName string, parameterContent interface{}) ([]map[string]interface{}, error) { - parameterNameSlice := strings.Split(parameterName, "-") - var parameterSlice []map[string]interface{} - parameterContentSlice := reflect.ValueOf(parameterContent) - if parameterContentSlice.Kind() != reflect.Slice { - return nil, errors.New("parameterContent should be slice") - } - for i := 0; i < parameterContentSlice.Len(); i++ { - parameterMap := make(map[string]interface{}) - elem := reflect.ValueOf(parameterContentSlice.Index(i).Interface()) - switch elem.Kind() { - case reflect.Map: - // e.g. "username-password": [{"username": "test1", "password": "passwd1", "other": "111"}, {"username": "test2", "password": "passwd2", "other": ""222}] - // -> [{"username": "test1", "password": "passwd1"}, {"username": "test2", "password": "passwd2"}] - for _, key := range parameterNameSlice { - if _, ok := elem.Interface().(map[string]interface{})[key]; ok { - parameterMap[key] = elem.MapIndex(reflect.ValueOf(key)).Interface() - } else { - log.Error().Interface("parameterNameSlice", parameterNameSlice).Msg("[parseParameters] parameter name not found") - return nil, errors.New("parameter name not found") - } - } - case reflect.Slice: - // e.g. "username-password": [["test1", "passwd1"], ["test2", "passwd2"]] - // -> [{"username": "test1", "password": "passwd1"}, {"username": "test2", "password": "passwd2"}] - if len(parameterNameSlice) != elem.Len() { - log.Error().Interface("parameterNameSlice", parameterNameSlice).Interface("parameterContent", elem.Interface()).Msg("[parseParameters] parameter name slice and parameter content slice should have the same length") - return nil, errors.New("parameter name slice and parameter content slice should have the same length") - } else { - for j := 0; j < elem.Len(); j++ { - parameterMap[parameterNameSlice[j]] = elem.Index(j).Interface() - } - } - default: - // e.g. "app_version": [3.1, 3.0] - // -> [{"app_version": 3.1}, {"app_version": 3.0}] - if len(parameterNameSlice) != 1 { - log.Error().Interface("parameterNameSlice", parameterNameSlice).Msg("[parseParameters] parameter name slice should have only one element when parameter content is string") - return nil, errors.New("parameter name slice should have only one element when parameter content is string") - } - parameterMap[parameterNameSlice[0]] = elem.Interface() - } - parameterSlice = append(parameterSlice, parameterMap) - } - return parameterSlice, nil -} - -func initParameterIterator(cfg *TConfig, mode string) (err error) { - var parameters map[string]iteratorParamsType - parameters, err = parseParameters(cfg.Parameters, cfg.Variables) - if err != nil { - return err - } - // parse config parameters setting - if cfg.ParametersSetting == nil { - cfg.ParametersSetting = &TParamsConfig{Iterators: []*Iterator{}} - } - // boomer模式下不限制迭代次数 - if mode == "boomer" { - cfg.ParametersSetting.Iteration = -1 - } - rawValue := reflect.ValueOf(cfg.ParametersSetting.Strategy) - switch rawValue.Kind() { - case reflect.Map: - // strategy: {"user_agent": "sequential", "username-password": "random"}, 每个参数对应一个迭代器，每个迭代器随机、顺序选取元素互不影响 - for k, v := range parameters { - if _, ok := rawValue.Interface().(map[string]interface{})[k]; ok { - // use strategy if configured - cfg.ParametersSetting.Iterators = append( - cfg.ParametersSetting.Iterators, - newIterator(v, iteratorStrategyType(rawValue.MapIndex(reflect.ValueOf(k)).String()), cfg.ParametersSetting.Iteration), - ) - } else { - // use sequential strategy by default - cfg.ParametersSetting.Iterators = append( - cfg.ParametersSetting.Iterators, - newIterator(v, strategySequential, cfg.ParametersSetting.Iteration), - ) - } - } - case reflect.String: - // strategy: random, 仅生成一个的迭代器，该迭代器在参数笛卡尔积slice中随机选取元素 - if len(rawValue.String()) == 0 { - cfg.ParametersSetting.Strategy = strategySequential - } else { - cfg.ParametersSetting.Strategy = iteratorStrategyType(strings.ToLower(rawValue.String())) - } - cfg.ParametersSetting.Iterators = append( - cfg.ParametersSetting.Iterators, - newIterator(genCartesianProduct(parameters), cfg.ParametersSetting.Strategy.(iteratorStrategyType), cfg.ParametersSetting.Iteration), - ) - default: - // default strategy: sequential, 仅生成一个的迭代器，该迭代器在参数笛卡尔积slice中顺序选取元素 - cfg.ParametersSetting.Strategy = strategySequential - cfg.ParametersSetting.Iterators = append( - cfg.ParametersSetting.Iterators, - newIterator(genCartesianProduct(parameters), cfg.ParametersSetting.Strategy.(iteratorStrategyType), cfg.ParametersSetting.Iteration), - ) - } - return nil -} - -func newIterator(parameters iteratorParamsType, strategy iteratorStrategyType, iteration int) *Iterator { - iter := parameters.Iterator() - iter.strategy = strategy - if iteration > 0 { - iter.iteration = iteration - } else if iteration < 0 { - iter.iteration = -1 - } else if iter.iteration == 0 { - iter.iteration = 1 - } - return iter -} diff --git a/hrp/parser_test.go b/hrp/parser_test.go index 38fa5cc9..10e843a7 100644 --- a/hrp/parser_test.go +++ b/hrp/parser_test.go @@ -1,7 +1,6 @@ package hrp import ( - "fmt" "sort" "testing" "time" @@ -754,152 +753,3 @@ func TestFindallVariables(t *testing.T) { } } } - -func TestParseParameters(t *testing.T) { - testData := []struct { - rawVars map[string]interface{} - expectLength int - }{ - { - map[string]interface{}{ - "username-password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), - "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}, - }, - 6, - }, - { - map[string]interface{}{ - "username-password": [][]interface{}{ - {"test1", "111111"}, - {"test2", "222222"}, - {"test3", "333333"}, - }, - "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}, - "app_version": []interface{}{0.3}, - }, - 6, - }, - { - map[string]interface{}{ - "username-password": [][]interface{}{ - {"test1", "111111"}, - {"test2", "222222"}, - {"test3", "333333"}, - }, - "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}, - "app_version": []interface{}{0.3, 0.4, 0.5}, - }, - 18, - }, - { - map[string]interface{}{}, - 0, - }, - { - nil, - 0, - }, - } - for _, data := range testData { - params, _ := parseParameters(data.rawVars, map[string]interface{}{}) - value := genCartesianProduct(params) - if !assert.Len(t, value, data.expectLength) { - t.Fail() - } - } -} - -func TestParseParametersError(t *testing.T) { - testData := []struct { - rawVars map[string]interface{} - }{ - { - map[string]interface{}{ - "username_password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), - "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}}, - }, - { - map[string]interface{}{ - "username-password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), - "user-agent": []interface{}{"IOS/10.1", "IOS/10.2"}}, - }, - { - map[string]interface{}{ - "username-password": fmt.Sprintf("${param(%s/account.csv)}", hrpExamplesDir), - "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}}, - }, - } - for _, data := range testData { - _, err := parseParameters(data.rawVars, map[string]interface{}{}) - if !assert.Error(t, err) { - t.Fail() - } - } -} - -func TestParseSlice(t *testing.T) { - testData := []struct { - rawVar1 string - rawVar2 interface{} - expect []map[string]interface{} - }{ - { - "username-password", - []map[string]interface{}{ - {"username": "test1", "password": 111111, "other": "111"}, - {"username": "test2", "password": 222222, "other": "222"}, - }, - []map[string]interface{}{ - {"username": "test1", "password": 111111}, - {"username": "test2", "password": 222222}, - }, - }, - { - "username-password", - [][]string{ - {"test1", "111111"}, - {"test2", "222222"}, - }, - []map[string]interface{}{ - {"username": "test1", "password": "111111"}, - {"username": "test2", "password": "222222"}, - }, - }, - { - "app_version", - []float64{3.1, 3.0}, - []map[string]interface{}{ - {"app_version": 3.1}, - {"app_version": 3.0}, - }, - }, - } - for _, data := range testData { - value, _ := parseSlice(data.rawVar1, data.rawVar2) - if !assert.Equal(t, data.expect, value) { - t.Fail() - } - } -} - -func TestParseSliceError(t *testing.T) { - testData := []struct { - rawVar1 string - rawVar2 interface{} - }{ - { - "app_version", - 123, - }, - { - "app_version", - "123", - }, - } - for _, data := range testData { - _, err := parseSlice(data.rawVar1, data.rawVar2) - if !assert.Error(t, err) { - t.Fail() - } - } -} diff --git a/hrp/runner.go b/hrp/runner.go index 2e751fd6..76da1e75 100644 --- a/hrp/runner.go +++ b/hrp/runner.go @@ -163,16 +163,8 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error { } }() - // 在runner模式下，指定整体策略，cfg.ParametersSetting.Iterators仅包含一个CartesianProduct的迭代器 - for it := sessionRunner.parsedConfig.ParametersSetting.Iterators[0]; it.HasNext(); { - var parameterVariables map[string]interface{} - // iterate through all parameter iterators and update case variables - for _, it := range sessionRunner.parsedConfig.ParametersSetting.Iterators { - if it.HasNext() { - parameterVariables = it.Next() - } - } - if err = sessionRunner.Start(parameterVariables); err != nil { + for it := sessionRunner.parametersIterator; it.HasNext(); { + if err = sessionRunner.Start(it.Next()); err != nil { log.Error().Err(err).Msg("[Run] run testcase failed") return err } diff --git a/hrp/session.go b/hrp/session.go index 478fbe03..fa4dbfa0 100644 --- a/hrp/session.go +++ b/hrp/session.go @@ -12,11 +12,12 @@ import ( // SessionRunner is used to run testcase and its steps. // each testcase has its own SessionRunner instance and share session variables. type SessionRunner struct { - testCase *TestCase - hrpRunner *HRPRunner - parser *Parser - parsedConfig *TConfig - sessionVariables map[string]interface{} + testCase *TestCase + hrpRunner *HRPRunner + parser *Parser + parsedConfig *TConfig + parametersIterator *ParametersIterator + sessionVariables map[string]interface{} // transactions stores transaction timing info. // key is transaction name, value is map of transaction type and time, e.g. start time and end time. transactions map[string]map[transactionType]time.Time @@ -61,13 +62,16 @@ func (r *SessionRunner) Start(givenVars map[string]interface{}) error { Str("type", string(step.Type())).Msg("run step start") stepResult, err := step.Run(r) - if err != nil && r.hrpRunner.failfast { + if err != nil { log.Error(). Str("step", stepResult.Name). Str("type", string(stepResult.StepType)). Bool("success", false). Msg("run step end") - return errors.Wrap(err, "abort running due to failfast setting") + + if r.hrpRunner.failfast { + return errors.Wrap(err, "abort running due to failfast setting") + } } // update extracted variables @@ -150,11 +154,15 @@ func (r *SessionRunner) parseConfig() error { r.parsedConfig.ThinkTimeSetting.checkThinkTime() // parse testcase config parameters - err = initParameterIterator(r.parsedConfig, "runner") + parametersIterator, err := initParametersIterator(r.parsedConfig) if err != nil { - log.Error().Interface("parameters", r.parsedConfig.Parameters).Err(err).Msg("parse config parameters failed") + log.Error().Err(err). + Interface("parameters", r.parsedConfig.Parameters). + Interface("parametersSetting", r.parsedConfig.ParametersSetting). + Msg("parse config parameters failed") return errors.Wrap(err, "parse testcase config parameters failed") } + r.parametersIterator = parametersIterator return nil } From 8ebb3634fb9fbaacec2b65a5bbae16ad74b89ed7 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 16 Apr 2022 00:36:38 +0800 Subject: [PATCH 2/7] fix: unittest with failfast --- hrp/internal/boomer/runner_test.go | 2 +- hrp/internal/boomer/stats_test.go | 2 +- hrp/internal/builtin/assertion_test.go | 20 ++--- hrp/internal/builtin/utils.go | 3 +- hrp/internal/har2case/core_test.go | 94 ++++++++++----------- hrp/internal/scaffold/examples_test.go | 6 +- hrp/internal/scaffold/templates/pytest.ini | 6 ++ hrp/internal/sdk/client_test.go | 2 +- hrp/parser.go | 2 +- hrp/parser_test.go | 96 +++++++++++----------- hrp/plugin_test.go | 14 ++-- hrp/response_test.go | 8 +- hrp/runner_test.go | 24 +++--- hrp/testcase_test.go | 22 ++--- 14 files changed, 154 insertions(+), 147 deletions(-) create mode 100644 hrp/internal/scaffold/templates/pytest.ini diff --git a/hrp/internal/boomer/runner_test.go b/hrp/internal/boomer/runner_test.go index a4f674b4..549980c9 100644 --- a/hrp/internal/boomer/runner_test.go +++ b/hrp/internal/boomer/runner_test.go @@ -109,6 +109,6 @@ func TestLoopCount(t *testing.T) { go runner.start() <-runner.stopChan if !assert.Equal(t, runner.loop.loopCount, atomic.LoadInt64(&runner.loop.finishedCount)) { - t.Fail() + t.Fatal() } } diff --git a/hrp/internal/boomer/stats_test.go b/hrp/internal/boomer/stats_test.go index afe0b41e..1d4806a2 100644 --- a/hrp/internal/boomer/stats_test.go +++ b/hrp/internal/boomer/stats_test.go @@ -155,7 +155,7 @@ func TestSerializeStats(t *testing.T) { first := serialized[0] entry, err := deserializeStatsEntry(first) if err != nil { - t.Fail() + t.Fatal() } if entry.Name != "success" { diff --git a/hrp/internal/builtin/assertion_test.go b/hrp/internal/builtin/assertion_test.go index d919464e..aa7d4bde 100644 --- a/hrp/internal/builtin/assertion_test.go +++ b/hrp/internal/builtin/assertion_test.go @@ -20,7 +20,7 @@ func TestStartsWith(t *testing.T) { for _, data := range testData { if !assert.True(t, StartsWith(t, data.raw, data.expected)) { - t.Fail() + t.Fatal() } } } @@ -38,7 +38,7 @@ func TestEndsWith(t *testing.T) { for _, data := range testData { if !assert.True(t, EndsWith(t, data.raw, data.expected)) { - t.Fail() + t.Fatal() } } } @@ -58,7 +58,7 @@ func TestEqualLength(t *testing.T) { for _, data := range testData { if !assert.True(t, EqualLength(t, data.raw, data.expected)) { - t.Fail() + t.Fatal() } } } @@ -78,7 +78,7 @@ func TestLessThanLength(t *testing.T) { for _, data := range testData { if !assert.True(t, LessThanLength(t, data.raw, data.expected)) { - t.Fail() + t.Fatal() } } } @@ -98,7 +98,7 @@ func TestLessOrEqualsLength(t *testing.T) { for _, data := range testData { if !assert.True(t, LessOrEqualsLength(t, data.raw, data.expected)) { - t.Fail() + t.Fatal() } } } @@ -115,7 +115,7 @@ func TestGreaterThanLength(t *testing.T) { for _, data := range testData { if !assert.True(t, GreaterThanLength(t, data.raw, data.expected)) { - t.Fail() + t.Fatal() } } } @@ -135,7 +135,7 @@ func TestGreaterOrEqualsLength(t *testing.T) { for _, data := range testData { if !assert.True(t, GreaterOrEqualsLength(t, data.raw, data.expected)) { - t.Fail() + t.Fatal() } } } @@ -152,7 +152,7 @@ func TestContainedBy(t *testing.T) { for _, data := range testData { if !assert.True(t, ContainedBy(t, data.raw, data.expected)) { - t.Fail() + t.Fatal() } } } @@ -169,7 +169,7 @@ func TestStringEqual(t *testing.T) { for _, data := range testData { if !assert.True(t, StringEqual(t, data.raw, data.expected)) { - t.Fail() + t.Fatal() } } } @@ -185,7 +185,7 @@ func TestRegexMatch(t *testing.T) { for _, data := range testData { if !assert.True(t, RegexMatch(t, data.raw, data.expected)) { - t.Fail() + t.Fatal() } } } diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index 47654912..4d4dfb56 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -270,11 +270,12 @@ func loadFromCSV(path string) []map[string]interface{} { log.Error().Err(err).Msg("parse csv file failed") os.Exit(1) } + firstLine := content[0] // parameter names var result []map[string]interface{} for i := 1; i < len(content); i++ { row := make(map[string]interface{}) for j := 0; j < len(content[i]); j++ { - row[content[0][j]] = content[i][j] + row[firstLine[j]] = content[i][j] } result = append(result, row) } diff --git a/hrp/internal/har2case/core_test.go b/hrp/internal/har2case/core_test.go index 93fb5ef6..fae4a3f5 100644 --- a/hrp/internal/har2case/core_test.go +++ b/hrp/internal/har2case/core_test.go @@ -17,20 +17,20 @@ var ( func TestGenJSON(t *testing.T) { jsonPath, err := NewHAR(harPath).GenJSON() if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.NotEmpty(t, jsonPath) { - t.Fail() + t.Fatal() } } func TestGenYAML(t *testing.T) { yamlPath, err := NewHAR(harPath2).GenYAML() if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.NotEmpty(t, yamlPath) { - t.Fail() + t.Fatal() } } @@ -38,13 +38,13 @@ func TestLoadHAR(t *testing.T) { har := NewHAR(harPath) h, err := har.load() if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, "GET", h.Log.Entries[0].Request.Method) { - t.Fail() + t.Fatal() } if !assert.Equal(t, "POST", h.Log.Entries[1].Request.Method) { - t.Fail() + t.Fatal() } } @@ -53,18 +53,18 @@ func TestLoadHARWithProfile(t *testing.T) { har.SetProfile(profilePath) _, err := har.load() if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, map[string]interface{}{"Content-Type": "application/x-www-form-urlencoded"}, har.profile["headers"]) { - t.Fail() + t.Fatal() } if !assert.Equal(t, map[string]interface{}{"UserName": "debugtalk"}, har.profile["cookies"]) { - t.Fail() + t.Fatal() } } @@ -72,73 +72,73 @@ func TestMakeTestCase(t *testing.T) { har := NewHAR(harPath) tCase, err := har.makeTestCase() if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } // make request method if !assert.EqualValues(t, "GET", tCase.TestSteps[0].Request.Method) { - t.Fail() + t.Fatal() } if !assert.EqualValues(t, "POST", tCase.TestSteps[1].Request.Method) { - t.Fail() + t.Fatal() } // make request url if !assert.Equal(t, "https://postman-echo.com/get", tCase.TestSteps[0].Request.URL) { - t.Fail() + t.Fatal() } if !assert.Equal(t, "https://postman-echo.com/post", tCase.TestSteps[1].Request.URL) { - t.Fail() + t.Fatal() } // make request params if !assert.Equal(t, "HDnY8", tCase.TestSteps[0].Request.Params["foo1"]) { - t.Fail() + t.Fatal() } // make request cookies if !assert.NotEmpty(t, tCase.TestSteps[1].Request.Cookies["sails.sid"]) { - t.Fail() + t.Fatal() } // make request headers if !assert.Equal(t, "HttpRunnerPlus", tCase.TestSteps[0].Request.Headers["User-Agent"]) { - t.Fail() + t.Fatal() } if !assert.Equal(t, "postman-echo.com", tCase.TestSteps[0].Request.Headers["Host"]) { - t.Fail() + t.Fatal() } // make request data if !assert.Equal(t, nil, tCase.TestSteps[0].Request.Body) { - t.Fail() + t.Fatal() } if !assert.Equal(t, map[string]interface{}{"foo1": "HDnY8", "foo2": 12.3}, tCase.TestSteps[1].Request.Body) { - t.Fail() + t.Fatal() } if !assert.Equal(t, "foo1=HDnY8&foo2=12.3", tCase.TestSteps[2].Request.Body) { - t.Fail() + t.Fatal() } // make validators validator, ok := tCase.TestSteps[0].Validators[0].(hrp.Validator) if !ok || !assert.Equal(t, "status_code", validator.Check) { - t.Fail() + t.Fatal() } validator, ok = tCase.TestSteps[0].Validators[1].(hrp.Validator) if !ok || !assert.Equal(t, "headers.\"Content-Type\"", validator.Check) { - t.Fail() + t.Fatal() } validator, ok = tCase.TestSteps[0].Validators[2].(hrp.Validator) if !ok || !assert.Equal(t, "body.url", validator.Check) { - t.Fail() + t.Fatal() } } func TestGetFilenameWithoutExtension(t *testing.T) { filename := getFilenameWithoutExtension(harPath2) if !assert.Equal(t, "postman-echo", filename) { - t.Fail() + t.Fatal() } } @@ -154,13 +154,13 @@ func TestMakeRequestHeaders(t *testing.T) { } step, err := har.prepareTestStep(entry) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, map[string]string{ "Content-Type": "application/json; charset=utf-8", }, step.Request.Headers) { - t.Fail() + t.Fatal() } } @@ -177,13 +177,13 @@ func TestMakeRequestHeadersWithProfile(t *testing.T) { } step, err := har.prepareTestStep(entry) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, map[string]string{ "Content-Type": "application/x-www-form-urlencoded", }, step.Request.Headers) { - t.Fail() + t.Fatal() } } @@ -200,14 +200,14 @@ func TestMakeRequestCookies(t *testing.T) { } step, err := har.prepareTestStep(entry) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, map[string]string{ "abc": "123", "UserName": "leolee", }, step.Request.Cookies) { - t.Fail() + t.Fatal() } } @@ -225,13 +225,13 @@ func TestMakeRequestCookiesWithProfile(t *testing.T) { } step, err := har.prepareTestStep(entry) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, map[string]string{ "UserName": "debugtalk", }, step.Request.Cookies) { - t.Fail() + t.Fatal() } } @@ -251,11 +251,11 @@ func TestMakeRequestDataParams(t *testing.T) { } step, err := har.prepareTestStep(entry) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, "a=1&b=2", step.Request.Body) { - t.Fail() + t.Fatal() } } @@ -272,11 +272,11 @@ func TestMakeRequestDataJSON(t *testing.T) { } step, err := har.prepareTestStep(entry) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, map[string]interface{}{"a": "1", "b": "2"}, step.Request.Body) { - t.Fail() + t.Fatal() } } @@ -293,11 +293,11 @@ func TestMakeRequestDataTextEmpty(t *testing.T) { } step, err := har.prepareTestStep(entry) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, nil, step.Request.Body) { // TODO - t.Fail() + t.Fatal() } } @@ -320,11 +320,11 @@ func TestMakeValidate(t *testing.T) { } step, err := har.prepareTestStep(entry) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } validator, ok := step.Validators[0].(hrp.Validator) if !ok { - t.Fail() + t.Fatal() } if !assert.Equal(t, validator, hrp.Validator{ @@ -332,12 +332,12 @@ func TestMakeValidate(t *testing.T) { Expect: 200, Assert: "equals", Message: "assert response status code"}) { - t.Fail() + t.Fatal() } validator, ok = step.Validators[1].(hrp.Validator) if !ok { - t.Fail() + t.Fatal() } if !assert.Equal(t, validator, hrp.Validator{ @@ -345,12 +345,12 @@ func TestMakeValidate(t *testing.T) { Expect: "application/json; charset=utf-8", Assert: "equals", Message: "assert response header Content-Type"}) { - t.Fail() + t.Fatal() } validator, ok = step.Validators[2].(hrp.Validator) if !ok { - t.Fail() + t.Fatal() } if !assert.Equal(t, validator, hrp.Validator{ @@ -358,6 +358,6 @@ func TestMakeValidate(t *testing.T) { Expect: float64(200), // TODO Assert: "equals", Message: "assert response body Code"}) { - t.Fail() + t.Fatal() } } diff --git a/hrp/internal/scaffold/examples_test.go b/hrp/internal/scaffold/examples_test.go index 35ec4381..22daa768 100644 --- a/hrp/internal/scaffold/examples_test.go +++ b/hrp/internal/scaffold/examples_test.go @@ -10,20 +10,20 @@ func TestGenDemoExamples(t *testing.T) { os.RemoveAll(dir) err := CreateScaffold(dir, Go) if err != nil { - t.Fail() + t.Fatal() } dir = "../../../examples/demo-with-py-plugin" os.RemoveAll(dir) err = CreateScaffold(dir, Py) if err != nil { - t.Fail() + t.Fatal() } dir = "../../../examples/demo-without-plugin" os.RemoveAll(dir) err = CreateScaffold(dir, Ignore) if err != nil { - t.Fail() + t.Fatal() } } diff --git a/hrp/internal/scaffold/templates/pytest.ini b/hrp/internal/scaffold/templates/pytest.ini new file mode 100644 index 00000000..62fcbe9b --- /dev/null +++ b/hrp/internal/scaffold/templates/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +addopts = -s +# https://docs.pytest.org/en/latest/how-to/output.html +junit_logging = all +junit_duration_report = total +log_cli = False \ No newline at end of file diff --git a/hrp/internal/sdk/client_test.go b/hrp/internal/sdk/client_test.go index 8c31cf47..e1044d47 100644 --- a/hrp/internal/sdk/client_test.go +++ b/hrp/internal/sdk/client_test.go @@ -25,6 +25,6 @@ func TestStructToUrlValues(t *testing.T) { } val := structToUrlValues(event) if val.Encode() != "ea=convert&ec=unittest&el=v0.3.0&ev=123" { - t.Fail() + t.Fatal() } } diff --git a/hrp/parser.go b/hrp/parser.go index 3a0d03bb..6941eaf7 100644 --- a/hrp/parser.go +++ b/hrp/parser.go @@ -454,7 +454,7 @@ func (p *Parser) ParseVariables(variables map[string]interface{}) (map[string]in return parsedVariables, nil } -type variableSet map[string]struct{} +type variableSet map[string]struct{} // TODO func extractVariables(raw interface{}) variableSet { rawValue := reflect.ValueOf(raw) diff --git a/hrp/parser_test.go b/hrp/parser_test.go index 10e843a7..ee0b6fb3 100644 --- a/hrp/parser_test.go +++ b/hrp/parser_test.go @@ -13,41 +13,41 @@ func TestBuildURL(t *testing.T) { url = buildURL("https://postman-echo.com", "/get") if !assert.Equal(t, url, "https://postman-echo.com/get") { - t.Fail() + t.Fatal() } url = buildURL("https://postman-echo.com", "get") if !assert.Equal(t, url, "https://postman-echo.com/get") { - t.Fail() + t.Fatal() } url = buildURL("https://postman-echo.com/", "/get") if !assert.Equal(t, url, "https://postman-echo.com/get") { - t.Fail() + t.Fatal() } url = buildURL("https://postman-echo.com/abc/", "/get?a=1&b=2") if !assert.Equal(t, url, "https://postman-echo.com/abc/get?a=1&b=2") { - t.Fail() + t.Fatal() } url = buildURL("https://postman-echo.com/abc", "get?a=1&b=2") if !assert.Equal(t, url, "https://postman-echo.com/abc/get?a=1&b=2") { - t.Fail() + t.Fatal() } // omit query string in base url url = buildURL("https://postman-echo.com/abc?x=6&y=9", "/get?a=1&b=2") if !assert.Equal(t, url, "https://postman-echo.com/abc/get?a=1&b=2") { - t.Fail() + t.Fatal() } url = buildURL("", "https://postman-echo.com/get") if !assert.Equal(t, url, "https://postman-echo.com/get") { - t.Fail() + t.Fatal() } // notice: step request url > config base url url = buildURL("https://postman-echo.com", "https://httpbin.org/get") if !assert.Equal(t, url, "https://httpbin.org/get") { - t.Fail() + t.Fatal() } } @@ -64,7 +64,7 @@ func TestRegexCompileVariable(t *testing.T) { for _, expr := range testData { varMatched := regexCompileVariable.FindStringSubmatch(expr) if !assert.Len(t, varMatched, 3) { - t.Fail() + t.Fatal() } } } @@ -81,7 +81,7 @@ func TestRegexCompileAbnormalVariable(t *testing.T) { for _, expr := range testData { varMatched := regexCompileVariable.FindStringSubmatch(expr) if !assert.Len(t, varMatched, 0) { - t.Fail() + t.Fatal() } } } @@ -99,7 +99,7 @@ func TestRegexCompileFunction(t *testing.T) { for _, expr := range testData { varMatched := regexCompileFunction.FindStringSubmatch(expr) if !assert.Len(t, varMatched, 3) { - t.Fail() + t.Fatal() } } } @@ -121,7 +121,7 @@ func TestRegexCompileAbnormalFunction(t *testing.T) { for _, expr := range testData { varMatched := regexCompileFunction.FindStringSubmatch(expr) if !assert.Len(t, varMatched, 0) { - t.Fail() + t.Fatal() } } } @@ -183,10 +183,10 @@ func TestParseDataStringWithVariables(t *testing.T) { for _, data := range testData { parsedData, err := parser.Parse(data.expr, variablesMapping) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expect, parsedData) { - t.Fail() + t.Fatal() } } } @@ -208,10 +208,10 @@ func TestParseDataStringWithUndefinedVariables(t *testing.T) { for _, data := range testData { parsedData, err := parser.Parse(data.expr, variablesMapping) if !assert.Error(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expect, parsedData) { - t.Fail() + t.Fatal() } } } @@ -253,10 +253,10 @@ func TestParseDataStringWithVariablesAbnormal(t *testing.T) { for _, data := range testData { parsedData, err := parser.Parse(data.expr, variablesMapping) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expect, parsedData) { - t.Fail() + t.Fatal() } } } @@ -284,10 +284,10 @@ func TestParseDataMapWithVariables(t *testing.T) { for _, data := range testData { parsedData, err := parser.Parse(data.expr, variablesMapping) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expect, parsedData) { - t.Fail() + t.Fatal() } } } @@ -318,10 +318,10 @@ func TestParseHeaders(t *testing.T) { for _, data := range testData { parsedHeaders, err := parser.ParseHeaders(data.rawHeaders, variablesMapping) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expectHeaders, parsedHeaders) { - t.Fail() + t.Fatal() } } } @@ -347,7 +347,7 @@ func TestMergeVariables(t *testing.T) { for _, data := range testData { mergedVariables := mergeVariables(data.stepVariables, data.configVariables) if !assert.Equal(t, data.expectVariables, mergedVariables) { - t.Fail() + t.Fatal() } } } @@ -383,7 +383,7 @@ func TestMergeMap(t *testing.T) { for _, data := range testData { mergedMap := mergeMap(data.m, data.overriddenMap) if !assert.Equal(t, data.expectMap, mergedMap) { - t.Fail() + t.Fatal() } } } @@ -414,7 +414,7 @@ func TestMergeSlices(t *testing.T) { for _, data := range testData { mergedSlice := mergeSlices(data.slice, data.overriddenSlice) if !assert.Equal(t, data.expectSlice, mergedSlice) { - t.Fail() + t.Fatal() } } } @@ -453,7 +453,7 @@ func TestMergeValidators(t *testing.T) { for _, data := range testData { mergedValidators := mergeValidators(data.validators, data.overriddenValidators) if !assert.Equal(t, data.expectValidators, mergedValidators) { - t.Fail() + t.Fatal() } } } @@ -464,35 +464,35 @@ func TestCallBuiltinFunction(t *testing.T) { // call function without arguments _, err := parser.CallFunc("get_timestamp") if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } // call function with one argument timeStart := time.Now() _, err = parser.CallFunc("sleep", 1) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Greater(t, time.Since(timeStart), time.Duration(1)*time.Second) { - t.Fail() + t.Fatal() } // call function with one argument result, err := parser.CallFunc("gen_random_string", 10) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, 10, len(result.(string))) { - t.Fail() + t.Fatal() } // call function with two argument result, err = parser.CallFunc("max", float64(10), 9.99) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, float64(10), result.(float64)) { - t.Fail() + t.Fatal() } } @@ -517,10 +517,10 @@ func TestLiteralEval(t *testing.T) { for _, data := range testData { value, err := literalEval(data.expr) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expect, value) { - t.Fail() + t.Fatal() } } } @@ -547,10 +547,10 @@ func TestParseFunctionArguments(t *testing.T) { for _, data := range testData { value, err := parseFunctionArguments(data.expr) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expect, value) { - t.Fail() + t.Fatal() } } } @@ -576,10 +576,10 @@ func TestParseDataStringWithFunctions(t *testing.T) { for _, data := range testData1 { value, err := parser.Parse(data.expr, variablesMapping) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expect, len(value.(string))) { - t.Fail() + t.Fatal() } } @@ -595,10 +595,10 @@ func TestParseDataStringWithFunctions(t *testing.T) { for _, data := range testData2 { value, err := parser.Parse(data.expr, variablesMapping) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expect, value) { - t.Fail() + t.Fatal() } } } @@ -619,7 +619,7 @@ func TestConvertString(t *testing.T) { for _, data := range testData { value := convertString(data.raw) if !assert.Equal(t, data.expect, value) { - t.Fail() + t.Fatal() } } } @@ -643,10 +643,10 @@ func TestParseVariables(t *testing.T) { for _, data := range testData { value, err := parser.ParseVariables(data.rawVars) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expectVars, value) { - t.Fail() + t.Fatal() } } } @@ -674,10 +674,10 @@ func TestParseVariablesAbnormal(t *testing.T) { for _, data := range testData { value, err := parser.ParseVariables(data.rawVars) if !assert.Error(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, data.expectVars, value) { - t.Fail() + t.Fatal() } } } @@ -710,7 +710,7 @@ func TestExtractVariables(t *testing.T) { } sort.Strings(varList) if !assert.Equal(t, data.expectVars, varList) { - t.Fail() + t.Fatal() } } } @@ -749,7 +749,7 @@ func TestFindallVariables(t *testing.T) { } sort.Strings(varList) if !assert.Equal(t, data.expectVars, varList) { - t.Fail() + t.Fatal() } } } diff --git a/hrp/plugin_test.go b/hrp/plugin_test.go index 612e40ae..26e74213 100644 --- a/hrp/plugin_test.go +++ b/hrp/plugin_test.go @@ -10,36 +10,36 @@ func TestLocateFile(t *testing.T) { // specify target file path _, err := locateFile(templatesDir+"plugin/debugtalk.go", "debugtalk.go") if !assert.Nil(t, err) { - t.Fail() + t.Fatal() } // specify path with the same dir _, err = locateFile(templatesDir+"plugin/debugtalk.py", "debugtalk.go") if !assert.Nil(t, err) { - t.Fail() + t.Fatal() } // specify target file path dir _, err = locateFile(templatesDir+"plugin/", "debugtalk.go") if !assert.Nil(t, err) { - t.Fail() + t.Fatal() } // specify wrong path _, err = locateFile(".", "debugtalk.go") if !assert.Error(t, err) { - t.Fail() + t.Fatal() } _, err = locateFile("/abc", "debugtalk.go") if !assert.Error(t, err) { - t.Fail() + t.Fatal() } } func TestLocatePythonPlugin(t *testing.T) { _, err := locatePlugin(templatesDir + "plugin/debugtalk.py") if !assert.Nil(t, err) { - t.Fail() + t.Fatal() } } @@ -49,6 +49,6 @@ func TestLocateGoPlugin(t *testing.T) { _, err := locatePlugin(templatesDir + "debugtalk.bin") if !assert.Nil(t, err) { - t.Fail() + t.Fatal() } } diff --git a/hrp/response_test.go b/hrp/response_test.go index 49a32aa2..5b06ba5b 100644 --- a/hrp/response_test.go +++ b/hrp/response_test.go @@ -24,11 +24,11 @@ func TestSearchJmespath(t *testing.T) { resp.Body = io.NopCloser(strings.NewReader(testText)) respObj, err := newResponseObject(t, newParser(), &resp) if err != nil { - t.Fail() + t.Fatal() } for _, data := range testData { if !assert.Equal(t, data.expected, respObj.searchJmespath(data.raw)) { - t.Fail() + t.Fatal() } } } @@ -49,11 +49,11 @@ func TestSearchRegexp(t *testing.T) { resp.Body = io.NopCloser(strings.NewReader(testText)) respObj, err := newResponseObject(t, newParser(), &resp) if err != nil { - t.Fail() + t.Fatal() } for _, data := range testData { if !assert.Equal(t, data.expected, respObj.searchRegexp(data.raw)) { - t.Fail() + t.Fatal() } } } diff --git a/hrp/runner_test.go b/hrp/runner_test.go index 5ea57e51..9510a975 100644 --- a/hrp/runner_test.go +++ b/hrp/runner_test.go @@ -161,7 +161,7 @@ func TestRunCaseWithPluginJSON(t *testing.T) { err := NewRunner(nil).Run(&demoTestCaseWithPluginJSONPath) // hrp.Run(testCase) if err != nil { - t.Fail() + t.Fatal() } } @@ -171,7 +171,7 @@ func TestRunCaseWithPluginYAML(t *testing.T) { err := NewRunner(nil).Run(&demoTestCaseWithPluginYAMLPath) // hrp.Run(testCase) if err != nil { - t.Fail() + t.Fatal() } } @@ -181,7 +181,7 @@ func TestRunCaseWithRefAPI(t *testing.T) { err := NewRunner(nil).Run(&demoTestCaseWithRefAPIPath) if err != nil { - t.Fail() + t.Fatal() } testcase := &TestCase{ @@ -195,7 +195,7 @@ func TestRunCaseWithRefAPI(t *testing.T) { r := NewRunner(t) err = r.Run(testcase) if err != nil { - t.Fail() + t.Fatal() } } @@ -204,30 +204,30 @@ func TestLoadTestCases(t *testing.T) { tc := TestCasePath("../examples/demo-with-py-plugin/testcases/") testCases, err := loadTestCases(&tc) if !assert.Nil(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, len(testCases), 3) { - t.Fail() + t.Fatal() } // load test cases from folder path, including sub folders tc = TestCasePath("../examples/demo-with-py-plugin/") testCases, err = loadTestCases(&tc) if !assert.Nil(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, len(testCases), 3) { - t.Fail() + t.Fatal() } // load test cases from single file path tc = demoTestCaseWithPluginJSONPath testCases, err = loadTestCases(&tc) if !assert.Nil(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, 1, len(testCases)) { - t.Fail() + t.Fatal() } // load test cases from TestCase instance @@ -236,9 +236,9 @@ func TestLoadTestCases(t *testing.T) { } testCases, err = loadTestCases(testcase) if !assert.Nil(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, len(testCases), 1) { - t.Fail() + t.Fatal() } } diff --git a/hrp/testcase_test.go b/hrp/testcase_test.go index 2b0556fb..e49eccfe 100644 --- a/hrp/testcase_test.go +++ b/hrp/testcase_test.go @@ -156,21 +156,21 @@ func TestGenDemoTestCase(t *testing.T) { tCase := demoTestCaseWithPlugin.ToTCase() err := builtin.Dump2JSON(tCase, demoTestCaseWithPluginJSONPath.GetPath()) if err != nil { - t.Fail() + t.Fatal() } err = builtin.Dump2YAML(tCase, demoTestCaseWithPluginYAMLPath.GetPath()) if err != nil { - t.Fail() + t.Fatal() } tCase = demoTestCaseWithoutPlugin.ToTCase() err = builtin.Dump2JSON(tCase, demoTestCaseWithoutPluginJSONPath.GetPath()) if err != nil { - t.Fail() + t.Fatal() } err = builtin.Dump2YAML(tCase, demoTestCaseWithoutPluginYAMLPath.GetPath()) if err != nil { - t.Fail() + t.Fatal() } } @@ -179,24 +179,24 @@ func TestLoadCase(t *testing.T) { tcYAML := &TCase{} err := builtin.LoadFile(demoTestCaseWithPluginJSONPath.GetPath(), tcJSON) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } err = builtin.LoadFile(demoTestCaseWithPluginYAMLPath.GetPath(), tcYAML) if !assert.NoError(t, err) { - t.Fail() + t.Fatal() } if !assert.Equal(t, tcJSON.Config.Name, tcYAML.Config.Name) { - t.Fail() + t.Fatal() } if !assert.Equal(t, tcJSON.Config.BaseURL, tcYAML.Config.BaseURL) { - t.Fail() + t.Fatal() } if !assert.Equal(t, tcJSON.TestSteps[1].Name, tcYAML.TestSteps[1].Name) { - t.Fail() + t.Fatal() } if !assert.Equal(t, tcJSON.TestSteps[1].Request, tcYAML.TestSteps[1].Request) { - t.Fail() + t.Fatal() } } @@ -222,7 +222,7 @@ func TestConvertCheckExpr(t *testing.T) { } for _, expr := range exprs { if !assert.Equal(t, convertCheckExpr(expr.before), expr.after) { - t.Fail() + t.Fatal() } } } From 30449b624a593c546137177ba09c0089ad3bf26f Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 16 Apr 2022 09:14:12 +0800 Subject: [PATCH 3/7] change: add unittests for init parameters iterator --- examples/hrp/parameters_test.json | 9 +- examples/hrp/parameters_test.yaml | 7 +- hrp/parameters.go | 33 ++++- hrp/parameters_test.go | 192 +++++++++++++++++++++++++++--- hrp/session.go | 9 +- 5 files changed, 223 insertions(+), 27 deletions(-) diff --git a/examples/hrp/parameters_test.json b/examples/hrp/parameters_test.json index f379f326..84c36531 100644 --- a/examples/hrp/parameters_test.json +++ b/examples/hrp/parameters_test.json @@ -6,18 +6,19 @@ "iOS/10.1", "iOS/10.2" ], - "username-password": "${parameterize(examples/hrp/account.csv)}" + "username-password": "${parameterize($file)}" }, "parameters_setting": { - "strategy": { + "strategies": { "user_agent": "sequential", "username-password": "random" }, - "iteration": 6 + "limit": 6 }, "variables": { "app_version": "v1", - "user_agent": "iOS/10.3" + "user_agent": "iOS/10.3", + "file": "examples/hrp/account.csv" }, "base_url": "https://postman-echo.com", "verify": false diff --git a/examples/hrp/parameters_test.yaml b/examples/hrp/parameters_test.yaml index 7c3ab523..b5d06c71 100644 --- a/examples/hrp/parameters_test.yaml +++ b/examples/hrp/parameters_test.yaml @@ -2,15 +2,16 @@ config: name: "request methods testcase: validate with parameters" parameters: user_agent: [ "iOS/10.1", "iOS/10.2" ] - username-password: ${parameterize(examples/hrp/account.csv)} + username-password: ${parameterize($file)} parameters_setting: - strategy: + strategies: user_agent: "sequential" username-password: "random" - iteration: 6 + limit: 6 variables: app_version: v1 user_agent: iOS/10.3 + file: examples/hrp/account.csv base_url: "https://postman-echo.com" verify: False diff --git a/hrp/parameters.go b/hrp/parameters.go index a42f87eb..9b263696 100644 --- a/hrp/parameters.go +++ b/hrp/parameters.go @@ -42,6 +42,9 @@ func initParametersIterator(cfg *TConfig) (*ParametersIterator, error) { } func newParametersIterator(parameters map[string]Parameters, config *TParamsConfig) *ParametersIterator { + if config == nil { + config = &TParamsConfig{} + } iterator := &ParametersIterator{ data: parameters, hasNext: true, @@ -52,7 +55,8 @@ func newParametersIterator(parameters map[string]Parameters, config *TParamsConf } if len(parameters) == 0 { - iterator.hasNext = false + iterator.data = map[string]Parameters{} + iterator.limit = 1 return iterator } @@ -75,12 +79,23 @@ func newParametersIterator(parameters map[string]Parameters, config *TParamsConf // generate cartesian product for sequential parameters iterator.sequentialParameters = genCartesianProduct(parametersList) + + if iterator.limit < 0 { + log.Warn().Msg("parameters unlimited mode is only supported for load testing") + iterator.limit = 0 + } if iterator.limit == 0 { + // limit not set if len(iterator.sequentialParameters) > 0 { + // use cartesian product of sequential parameters size as limit iterator.limit = len(iterator.sequentialParameters) } else { + // all parameters are selected by random + // only run once iterator.limit = 1 } + } else { // limit > 0 + log.Info().Int("limit", iterator.limit).Msg("set limit for parameters") } return iterator @@ -98,6 +113,7 @@ type ParametersIterator struct { // SetUnlimitedMode is used for load testing func (iter *ParametersIterator) SetUnlimitedMode() { + log.Info().Msg("set parameters unlimited mode") iter.limit = -1 } @@ -134,15 +150,24 @@ func (iter *ParametersIterator) Next() map[string]interface{} { return nil } - selectedParameters := make(map[string]interface{}) - if iter.index < len(iter.sequentialParameters) { + var selectedParameters map[string]interface{} + if len(iter.sequentialParameters) == 0 { + selectedParameters = make(map[string]interface{}) + } else if iter.index < len(iter.sequentialParameters) { selectedParameters = iter.sequentialParameters[iter.index] + } else { + // loop back to the first sequential parameter + index := iter.index % len(iter.sequentialParameters) + selectedParameters = iter.sequentialParameters[index] } + // merge with random parameters for _, paramName := range iter.randomParameterNames { randSource := rand.New(rand.NewSource(time.Now().Unix())) randIndex := randSource.Intn(len(iter.data[paramName])) - selectedParameters[paramName] = iter.data[paramName][randIndex] + for k, v := range iter.data[paramName][randIndex] { + selectedParameters[k] = v + } } iter.index++ diff --git a/hrp/parameters_test.go b/hrp/parameters_test.go index 5f834313..f54516c3 100644 --- a/hrp/parameters_test.go +++ b/hrp/parameters_test.go @@ -4,7 +4,6 @@ import ( "fmt" "testing" - "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" ) @@ -31,7 +30,7 @@ func TestLoadParameters(t *testing.T) { {"test1", "111111"}, {"test2", "222222"}, }, - "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}, + "user_agent": []interface{}{"iOS/10.1", "iOS/10.2"}, "app_version": []interface{}{4.0}, }, map[string]Parameters{ @@ -40,8 +39,8 @@ func TestLoadParameters(t *testing.T) { {"username": "test2", "password": "222222"}, }, "user_agent": { - {"user_agent": "IOS/10.1"}, - {"user_agent": "IOS/10.2"}, + {"user_agent": "iOS/10.1"}, + {"user_agent": "iOS/10.2"}, }, "app_version": { {"app_version": 4.0}, @@ -79,17 +78,17 @@ func TestLoadParametersError(t *testing.T) { { map[string]interface{}{ "username_password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), - "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}}, + "user_agent": []interface{}{"iOS/10.1", "iOS/10.2"}}, }, { map[string]interface{}{ "username-password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), - "user-agent": []interface{}{"IOS/10.1", "IOS/10.2"}}, + "user-agent": []interface{}{"iOS/10.1", "iOS/10.2"}}, }, { map[string]interface{}{ "username-password": fmt.Sprintf("${param(%s/account.csv)}", hrpExamplesDir), - "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}}, + "user_agent": []interface{}{"iOS/10.1", "iOS/10.2"}}, }, } for _, data := range testData { @@ -100,23 +99,69 @@ func TestLoadParametersError(t *testing.T) { } } -func TestInitParametersIterator(t *testing.T) { +func TestInitParametersIteratorCount(t *testing.T) { configParameters := map[string]interface{}{ "username-password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), // 3 - "user_agent": []interface{}{"IOS/10.1", "IOS/10.2"}, - "app_version": []interface{}{4.0}, + "user_agent": []interface{}{"iOS/10.1", "iOS/10.2"}, // 2 + "app_version": []interface{}{4.0}, // 1 } testData := []struct { cfg *TConfig expectLimit int }{ + // default, no parameters setting { &TConfig{ Parameters: configParameters, ParametersSetting: &TParamsConfig{}, }, - 6, + 6, // 3 * 2 * 1 }, + { + &TConfig{ + Parameters: configParameters, + }, + 6, // 3 * 2 * 1 + }, + // default equals to set overall parameters strategy to "sequential" + { + &TConfig{ + Parameters: configParameters, + ParametersSetting: &TParamsConfig{ + Strategy: "sequential", + }, + }, + 6, // 3 * 2 * 1 + }, + // default equals to set each individual parameters strategy to "sequential" + { + &TConfig{ + Parameters: configParameters, + ParametersSetting: &TParamsConfig{ + Strategies: map[string]iteratorStrategy{ + "username-password": "sequential", + "user_agent": "sequential", + "app_version": "sequential", + }, + }, + }, + 6, // 3 * 2 * 1 + }, + { + &TConfig{ + Parameters: configParameters, + ParametersSetting: &TParamsConfig{ + Strategies: map[string]iteratorStrategy{ + "user_agent": "sequential", + "app_version": "sequential", + }, + }, + }, + 6, // 3 * 2 * 1 + }, + + // set overall parameters overall strategy to "random" + // each random parameters only select one item { &TConfig{ Parameters: configParameters, @@ -124,7 +169,20 @@ func TestInitParametersIterator(t *testing.T) { Strategy: "random", }, }, - 1, + 1, // 1 * 1 * 1 + }, + // set some individual parameters strategy to "random" + // this will override overall strategy + { + &TConfig{ + Parameters: configParameters, + ParametersSetting: &TParamsConfig{ + Strategies: map[string]iteratorStrategy{ + "user_agent": "random", + }, + }, + }, + 3, // 3 * 1 * 1 }, { &TConfig{ @@ -135,7 +193,37 @@ func TestInitParametersIterator(t *testing.T) { }, }, }, - 2, + 2, // 1 * 2 * 1 + }, + + // set limit for parameters + { + &TConfig{ + Parameters: configParameters, // total: 6 = 3 * 2 * 1 + ParametersSetting: &TParamsConfig{ + Limit: 4, // limit could be less than total + }, + }, + 4, + }, + { + &TConfig{ + Parameters: configParameters, // total: 6 = 3 * 2 * 1 + ParametersSetting: &TParamsConfig{ + Limit: 9, // limit could also be greater than total + }, + }, + 9, + }, + + // no parameters + // also will generate one empty item + { + &TConfig{ + Parameters: nil, + ParametersSetting: nil, + }, + 1, }, } for _, data := range testData { @@ -151,7 +239,7 @@ func TestInitParametersIterator(t *testing.T) { if !assert.True(t, iterator.HasNext()) { t.Fatal() } - log.Info().Interface("next", iterator.Next()).Msg("get next parameters") + iterator.Next() // consume next parameters } // should not have next if !assert.False(t, iterator.HasNext()) { @@ -160,6 +248,82 @@ func TestInitParametersIterator(t *testing.T) { } } +func TestInitParametersIteratorContent(t *testing.T) { + configParameters := map[string]interface{}{ + "username-password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), // 3 + "user_agent": []interface{}{"iOS/10.1", "iOS/10.2"}, // 2 + "app_version": []interface{}{4.0}, // 1 + } + testData := []struct { + cfg *TConfig + checkIndex int + expectParameters map[string]interface{} + }{ + // default, no parameters setting + { + &TConfig{ + Parameters: configParameters, + }, + 0, // check first item + map[string]interface{}{ + "username": "test1", "password": "111111", "user_agent": "iOS/10.1", "app_version": 4.0, + }, + }, + + // set limit for parameters + { + &TConfig{ + Parameters: map[string]interface{}{ + "username-password": []map[string]interface{}{ // 1 + {"username": "test1", "password": 111111, "other": "111"}, + }, + "user_agent": []string{"iOS/10.1", "iOS/10.2"}, // 2 + }, + ParametersSetting: &TParamsConfig{ + Limit: 5, // limit could also be greater than total + Strategies: map[string]iteratorStrategy{ + "username-password": "random", + }, + }, + }, + 2, // check 3th item, equals to the first item + map[string]interface{}{ + "username": "test1", "password": 111111, "user_agent": "iOS/10.1", + }, + }, + + // no parameters + // also will generate one empty item + { + &TConfig{ + Parameters: nil, + ParametersSetting: nil, + }, + 0, + map[string]interface{}(nil), + }, + } + for _, data := range testData { + iterator, err := initParametersIterator(data.cfg) + if !assert.Nil(t, err) { + t.Fatal() + } + + // get expected parameters item + for i := 0; i < data.checkIndex; i++ { + if !assert.True(t, iterator.HasNext()) { + t.Fatal() + } + iterator.Next() // consume next parameters + } + parametersItem := iterator.Next() + + if !assert.Equal(t, data.expectParameters, parametersItem) { + t.Fatal() + } + } +} + func TestGenCartesianProduct(t *testing.T) { testData := []struct { multiParameters []Parameters diff --git a/hrp/session.go b/hrp/session.go index fa4dbfa0..8172d2ab 100644 --- a/hrp/session.go +++ b/hrp/session.go @@ -111,8 +111,13 @@ func (r *SessionRunner) MergeStepVariables(vars map[string]interface{}) (map[str // updateConfigVariables updates config variables with given variables. // this is used for data driven -func (r *SessionRunner) updateConfigVariables(givenVars map[string]interface{}) { - for k, v := range givenVars { +func (r *SessionRunner) updateConfigVariables(parameters map[string]interface{}) { + if parameters == nil { + return + } + + log.Info().Interface("parameters", parameters).Msg("update config variables") + for k, v := range parameters { r.parsedConfig.Variables[k] = v } } From e2f3dd726d24935c6c854d214defc41651aa6e83 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 16 Apr 2022 12:36:17 +0800 Subject: [PATCH 4/7] change: update docs --- docs/cmd/hrp.md | 14 ++++++++------ docs/cmd/hrp_boom.md | 4 ++-- docs/cmd/hrp_har2case.md | 4 ++-- docs/cmd/hrp_pytest.md | 19 +++++++++++++++++++ docs/cmd/hrp_run.md | 6 +++--- docs/cmd/hrp_startproject.md | 4 ++-- hrp/cmd/root.go | 2 +- 7 files changed, 37 insertions(+), 16 deletions(-) create mode 100644 docs/cmd/hrp_pytest.md diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index 203deeaa..7b19c12c 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -1,6 +1,6 @@ ## hrp -One-stop solution for HTTP(S) testing. +Next-Generation API Testing Solution. ### Synopsis @@ -12,12 +12,13 @@ One-stop solution for HTTP(S) testing. ██║ ██║ ██║ ██║ ██║ ██║ ██║╚██████╔╝██║ ╚████║██║ ╚████║███████╗██║ ██║ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝╚═╝ ╚═══╝╚══════╝╚═╝ ╚═╝ -hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, -load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨ +HttpRunner is an open source API testing tool that supports HTTP(S)/HTTP2/WebSocket/RPC +network protocols, covering API testing, performance testing and digital experience +monitoring (DEM) test types. Enjoy! ✨ 🚀 ✨ License: Apache-2.0 Website: https://httprunner.com -Github: https://github.com/httprunner/httprunner/hrp +Github: https://github.com/httprunner/httprunner Copyright 2021 debugtalk ### Options @@ -30,7 +31,8 @@ Copyright 2021 debugtalk * [hrp boom](hrp_boom.md) - run load test with boomer * [hrp har2case](hrp_har2case.md) - convert HAR to json/yaml testcase files -* [hrp run](hrp_run.md) - run API test +* [hrp pytest](hrp_pytest.md) - run API test with pytest +* [hrp run](hrp_run.md) - run API test with go engine * [hrp startproject](hrp_startproject.md) - create a scaffold project -###### Auto generated by spf13/cobra on 26-Mar-2022 +###### Auto generated by spf13/cobra on 16-Apr-2022 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index 3b172a65..0315dece 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -39,6 +39,6 @@ hrp boom [flags] ### SEE ALSO -* [hrp](hrp.md) - One-stop solution for HTTP(S) testing. +* [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 26-Mar-2022 +###### Auto generated by spf13/cobra on 16-Apr-2022 diff --git a/docs/cmd/hrp_har2case.md b/docs/cmd/hrp_har2case.md index 918f27aa..5fd11b46 100644 --- a/docs/cmd/hrp_har2case.md +++ b/docs/cmd/hrp_har2case.md @@ -22,6 +22,6 @@ hrp har2case $har_path... [flags] ### SEE ALSO -* [hrp](hrp.md) - One-stop solution for HTTP(S) testing. +* [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 26-Mar-2022 +###### Auto generated by spf13/cobra on 16-Apr-2022 diff --git a/docs/cmd/hrp_pytest.md b/docs/cmd/hrp_pytest.md new file mode 100644 index 00000000..3f181b01 --- /dev/null +++ b/docs/cmd/hrp_pytest.md @@ -0,0 +1,19 @@ +## hrp pytest + +run API test with pytest + +``` +hrp pytest $path ... [flags] +``` + +### Options + +``` + -h, --help help for pytest +``` + +### SEE ALSO + +* [hrp](hrp.md) - Next-Generation API Testing Solution. + +###### Auto generated by spf13/cobra on 16-Apr-2022 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index 9853458d..3576c306 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -1,6 +1,6 @@ ## hrp run -run API test +run API test with go engine ### Synopsis @@ -32,6 +32,6 @@ hrp run $path... [flags] ### SEE ALSO -* [hrp](hrp.md) - One-stop solution for HTTP(S) testing. +* [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 26-Mar-2022 +###### Auto generated by spf13/cobra on 16-Apr-2022 diff --git a/docs/cmd/hrp_startproject.md b/docs/cmd/hrp_startproject.md index 12039f93..d1c109bd 100644 --- a/docs/cmd/hrp_startproject.md +++ b/docs/cmd/hrp_startproject.md @@ -17,6 +17,6 @@ hrp startproject $project_name [flags] ### SEE ALSO -* [hrp](hrp.md) - One-stop solution for HTTP(S) testing. +* [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 26-Mar-2022 +###### Auto generated by spf13/cobra on 16-Apr-2022 diff --git a/hrp/cmd/root.go b/hrp/cmd/root.go index b0d69625..9128d37e 100644 --- a/hrp/cmd/root.go +++ b/hrp/cmd/root.go @@ -15,7 +15,7 @@ import ( // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "hrp", - Short: "One-stop solution for API testing.", + Short: "Next-Generation API Testing Solution.", Long: ` ██╗ ██╗████████╗████████╗██████╗ ██████╗ ██╗ ██╗███╗ ██╗███╗ ██╗███████╗██████╗ ██║ ██║╚══██╔══╝╚══██╔══╝██╔══██╗██╔══██╗██║ ██║████╗ ██║████╗ ██║██╔════╝██╔══██╗ From 75e3729f8a893f689af7c2de311776824533df4d Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 16 Apr 2022 13:25:39 +0800 Subject: [PATCH 5/7] change: add unittests for parameters iterator unlimited mode --- hrp/parameters.go | 11 +++------ hrp/parameters_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++ hrp/session.go | 2 +- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/hrp/parameters.go b/hrp/parameters.go index 9b263696..33631c2d 100644 --- a/hrp/parameters.go +++ b/hrp/parameters.go @@ -13,7 +13,7 @@ import ( type TParamsConfig struct { Strategy iteratorStrategy `json:"strategy,omitempty" yaml:"strategy,omitempty"` // overall strategy - Strategies map[string]iteratorStrategy `json:"strategies,omitempty" yaml:"strategies,omitempty"` // map[string]string、string + Strategies map[string]iteratorStrategy `json:"strategies,omitempty" yaml:"strategies,omitempty"` // individual strategies for each parameters Limit int `json:"limit,omitempty" yaml:"limit,omitempty"` } @@ -107,8 +107,8 @@ type ParametersIterator struct { hasNext bool // cache query result sequentialParameters Parameters // cartesian product for sequential parameters randomParameterNames []string // value is parameter names - limit int - index int + limit int // limit count for iteration + index int // current iteration index } // SetUnlimitedMode is used for load testing @@ -145,11 +145,6 @@ func (iter *ParametersIterator) Next() map[string]interface{} { return nil } - if len(iter.data) == 0 { - iter.hasNext = false - return nil - } - var selectedParameters map[string]interface{} if len(iter.sequentialParameters) == 0 { selectedParameters = make(map[string]interface{}) diff --git a/hrp/parameters_test.go b/hrp/parameters_test.go index f54516c3..e4bac50a 100644 --- a/hrp/parameters_test.go +++ b/hrp/parameters_test.go @@ -248,6 +248,59 @@ func TestInitParametersIteratorCount(t *testing.T) { } } +func TestInitParametersIteratorUnlimitedCount(t *testing.T) { + configParameters := map[string]interface{}{ + "username-password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), // 3 + "user_agent": []interface{}{"iOS/10.1", "iOS/10.2"}, // 2 + "app_version": []interface{}{4.0}, // 1 + } + testData := []struct { + cfg *TConfig + }{ + // default, no parameters setting + { + &TConfig{ + Parameters: configParameters, + ParametersSetting: &TParamsConfig{}, + }, + }, + + // no parameters + // also will generate one empty item + { + &TConfig{ + Parameters: nil, + ParametersSetting: nil, + }, + }, + } + for _, data := range testData { + iterator, err := initParametersIterator(data.cfg) + if !assert.Nil(t, err) { + t.Fatal() + } + // set unlimited mode + iterator.SetUnlimitedMode() + if !assert.Equal(t, -1, iterator.limit) { + t.Fatal() + } + + for i := 0; i < 100; i++ { + if !assert.True(t, iterator.HasNext()) { + t.Fatal() + } + iterator.Next() // consume next parameters + } + if !assert.Equal(t, 100, iterator.index) { + t.Fatal() + } + // should also have next + if !assert.True(t, iterator.HasNext()) { + t.Fatal() + } + } +} + func TestInitParametersIteratorContent(t *testing.T) { configParameters := map[string]interface{}{ "username-password": fmt.Sprintf("${parameterize(%s/account.csv)}", hrpExamplesDir), // 3 diff --git a/hrp/session.go b/hrp/session.go index 8172d2ab..05621d09 100644 --- a/hrp/session.go +++ b/hrp/session.go @@ -112,7 +112,7 @@ func (r *SessionRunner) MergeStepVariables(vars map[string]interface{}) (map[str // updateConfigVariables updates config variables with given variables. // this is used for data driven func (r *SessionRunner) updateConfigVariables(parameters map[string]interface{}) { - if parameters == nil { + if len(parameters) == 0 { return } From 8674adc36b8d5a9da4a6542d82f2dc4de81a20ab Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 16 Apr 2022 13:47:59 +0800 Subject: [PATCH 6/7] fix: data race by summary --- hrp/boomer.go | 17 ++++---- hrp/parameters_test.go | 2 +- hrp/runner.go | 89 ++++++++++++++++++++++++++++++++++++++++-- hrp/session.go | 60 ++-------------------------- 4 files changed, 99 insertions(+), 69 deletions(-) diff --git a/hrp/boomer.go b/hrp/boomer.go index 7bba9285..d639f981 100644 --- a/hrp/boomer.go +++ b/hrp/boomer.go @@ -72,15 +72,16 @@ func (b *HRPBoomer) Quit() { } func (b *HRPBoomer) convertBoomerTask(testcase *TestCase, rendezvousList []*Rendezvous) *boomer.Task { - // init session runner for testcase - sessionRunner, err := b.hrpRunner.NewSessionRunner(testcase) + // init runner for testcase + // this runner is shared by multiple session runners + caseRunner, err := b.hrpRunner.newCaseRunner(testcase) if err != nil { - log.Error().Err(err).Msg("failed to create session runner") + log.Error().Err(err).Msg("failed to create runner") os.Exit(1) } - if sessionRunner.parser.plugin != nil { + if caseRunner.parser.plugin != nil { b.pluginsMutex.Lock() - b.plugins = append(b.plugins, sessionRunner.parser.plugin) + b.plugins = append(b.plugins, caseRunner.parser.plugin) b.pluginsMutex.Unlock() } @@ -93,7 +94,7 @@ func (b *HRPBoomer) convertBoomerTask(testcase *TestCase, rendezvousList []*Rend }() // set paramters mode for load testing - parametersIterator := sessionRunner.parametersIterator + parametersIterator := caseRunner.parametersIterator parametersIterator.SetUnlimitedMode() return &boomer.Task{ @@ -103,11 +104,13 @@ func (b *HRPBoomer) convertBoomerTask(testcase *TestCase, rendezvousList []*Rend testcaseSuccess := true // flag whole testcase result transactionSuccess := true // flag current transaction result + // init session runner + sessionRunner := caseRunner.newSession() + if parametersIterator.HasNext() { sessionRunner.updateConfigVariables(parametersIterator.Next()) } - sessionRunner.resetSession() startTime := time.Now() for _, step := range testcase.TestSteps { stepResult, err := step.Run(sessionRunner) diff --git a/hrp/parameters_test.go b/hrp/parameters_test.go index e4bac50a..0276be91 100644 --- a/hrp/parameters_test.go +++ b/hrp/parameters_test.go @@ -353,7 +353,7 @@ func TestInitParametersIteratorContent(t *testing.T) { ParametersSetting: nil, }, 0, - map[string]interface{}(nil), + map[string]interface{}{}, }, } for _, data := range testData { diff --git a/hrp/runner.go b/hrp/runner.go index 76da1e75..acdda315 100644 --- a/hrp/runner.go +++ b/hrp/runner.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/jinzhu/copier" "github.com/pkg/errors" "github.com/rs/zerolog/log" "golang.org/x/net/http2" @@ -208,11 +209,23 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error { // NewSessionRunner creates a new session runner for testcase. // each testcase has its own session runner func (r *HRPRunner) NewSessionRunner(testcase *TestCase) (*SessionRunner, error) { + runner, err := r.newCaseRunner(testcase) + if err != nil { + return nil, err + } + sessionRunner := &SessionRunner{ + testCaseRunner: runner, + } + sessionRunner.resetSession() + return sessionRunner, nil +} + +func (r *HRPRunner) newCaseRunner(testcase *TestCase) (*testCaseRunner, error) { + runner := &testCaseRunner{ testCase: testcase, hrpRunner: r, parser: newParser(), - summary: newSummary(), } // init parser plugin @@ -220,12 +233,80 @@ func (r *HRPRunner) NewSessionRunner(testcase *TestCase) (*SessionRunner, error) if err != nil { return nil, errors.Wrap(err, "init plugin failed") } - sessionRunner.parser.plugin = plugin + runner.parser.plugin = plugin // parse testcase config - if err := sessionRunner.parseConfig(); err != nil { + if err := runner.parseConfig(); err != nil { return nil, errors.Wrap(err, "parse testcase config failed") } - return sessionRunner, nil + return runner, nil +} + +type testCaseRunner struct { + testCase *TestCase + hrpRunner *HRPRunner + parser *Parser + parsedConfig *TConfig + parametersIterator *ParametersIterator +} + +// parseConfig parses testcase config, stores to parsedConfig. +func (r *testCaseRunner) parseConfig() error { + cfg := r.testCase.Config + + r.parsedConfig = &TConfig{} + // deep copy config to avoid data racing + if err := copier.Copy(r.parsedConfig, cfg); err != nil { + log.Error().Err(err).Msg("copy testcase config failed") + return err + } + + // parse config variables + parsedVariables, err := r.parser.ParseVariables(cfg.Variables) + if err != nil { + log.Error().Interface("variables", cfg.Variables).Err(err).Msg("parse config variables failed") + return err + } + r.parsedConfig.Variables = parsedVariables + + // parse config name + parsedName, err := r.parser.ParseString(cfg.Name, parsedVariables) + if err != nil { + return errors.Wrap(err, "parse config name failed") + } + r.parsedConfig.Name = convertString(parsedName) + + // parse config base url + parsedBaseURL, err := r.parser.ParseString(cfg.BaseURL, parsedVariables) + if err != nil { + return errors.Wrap(err, "parse config base url failed") + } + r.parsedConfig.BaseURL = convertString(parsedBaseURL) + + // ensure correction of think time config + r.parsedConfig.ThinkTimeSetting.checkThinkTime() + + // parse testcase config parameters + parametersIterator, err := initParametersIterator(r.parsedConfig) + if err != nil { + log.Error().Err(err). + Interface("parameters", r.parsedConfig.Parameters). + Interface("parametersSetting", r.parsedConfig.ParametersSetting). + Msg("parse config parameters failed") + return errors.Wrap(err, "parse testcase config parameters failed") + } + r.parametersIterator = parametersIterator + + return nil +} + +// each boomer task initiates a new session +// in order to avoid data racing +func (r *testCaseRunner) newSession() *SessionRunner { + sessionRunner := &SessionRunner{ + testCaseRunner: r, + } + sessionRunner.resetSession() + return sessionRunner } diff --git a/hrp/session.go b/hrp/session.go index 05621d09..4d120cc5 100644 --- a/hrp/session.go +++ b/hrp/session.go @@ -4,7 +4,6 @@ import ( _ "embed" "time" - "github.com/jinzhu/copier" "github.com/pkg/errors" "github.com/rs/zerolog/log" ) @@ -12,12 +11,8 @@ import ( // SessionRunner is used to run testcase and its steps. // each testcase has its own SessionRunner instance and share session variables. type SessionRunner struct { - testCase *TestCase - hrpRunner *HRPRunner - parser *Parser - parsedConfig *TConfig - parametersIterator *ParametersIterator - sessionVariables map[string]interface{} + *testCaseRunner + sessionVariables map[string]interface{} // transactions stores transaction timing info. // key is transaction name, value is map of transaction type and time, e.g. start time and end time. transactions map[string]map[transactionType]time.Time @@ -30,6 +25,7 @@ func (r *SessionRunner) resetSession() { r.sessionVariables = make(map[string]interface{}) r.transactions = make(map[string]map[transactionType]time.Time) r.startTime = time.Now() + r.summary = newSummary() } func (r *SessionRunner) GetParser() *Parser { @@ -122,56 +118,6 @@ func (r *SessionRunner) updateConfigVariables(parameters map[string]interface{}) } } -// parseConfig parses testcase config, stores to parsedConfig. -func (r *SessionRunner) parseConfig() error { - cfg := r.testCase.Config - - r.parsedConfig = &TConfig{} - // deep copy config to avoid data racing - if err := copier.Copy(r.parsedConfig, cfg); err != nil { - log.Error().Err(err).Msg("copy testcase config failed") - return err - } - - // parse config variables - parsedVariables, err := r.parser.ParseVariables(cfg.Variables) - if err != nil { - log.Error().Interface("variables", cfg.Variables).Err(err).Msg("parse config variables failed") - return err - } - r.parsedConfig.Variables = parsedVariables - - // parse config name - parsedName, err := r.parser.ParseString(cfg.Name, parsedVariables) - if err != nil { - return errors.Wrap(err, "parse config name failed") - } - r.parsedConfig.Name = convertString(parsedName) - - // parse config base url - parsedBaseURL, err := r.parser.ParseString(cfg.BaseURL, parsedVariables) - if err != nil { - return errors.Wrap(err, "parse config base url failed") - } - r.parsedConfig.BaseURL = convertString(parsedBaseURL) - - // ensure correction of think time config - r.parsedConfig.ThinkTimeSetting.checkThinkTime() - - // parse testcase config parameters - parametersIterator, err := initParametersIterator(r.parsedConfig) - if err != nil { - log.Error().Err(err). - Interface("parameters", r.parsedConfig.Parameters). - Interface("parametersSetting", r.parsedConfig.ParametersSetting). - Msg("parse config parameters failed") - return errors.Wrap(err, "parse testcase config parameters failed") - } - r.parametersIterator = parametersIterator - - return nil -} - func (r *SessionRunner) GetSummary() *TestCaseSummary { caseSummary := r.summary caseSummary.Name = r.parsedConfig.Name From 220df1385a4874ed71a8e5353d682a5bf34264a5 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 17 Apr 2022 10:41:31 +0800 Subject: [PATCH 7/7] fix: gen html report path --- docs/cmd/hrp.md | 2 +- docs/cmd/hrp_boom.md | 2 +- docs/cmd/hrp_har2case.md | 2 +- docs/cmd/hrp_pytest.md | 2 +- docs/cmd/hrp_run.md | 2 +- docs/cmd/hrp_startproject.md | 2 +- hrp/plugin.go | 8 +++++--- hrp/runner.go | 22 ++++----------------- hrp/summary.go | 37 +++++++++++++++++++++++++++++------- 9 files changed, 45 insertions(+), 34 deletions(-) diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index 7b19c12c..1a3960a4 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -35,4 +35,4 @@ Copyright 2021 debugtalk * [hrp run](hrp_run.md) - run API test with go engine * [hrp startproject](hrp_startproject.md) - create a scaffold project -###### Auto generated by spf13/cobra on 16-Apr-2022 +###### Auto generated by spf13/cobra on 17-Apr-2022 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index 0315dece..7be66d74 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -41,4 +41,4 @@ hrp boom [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 16-Apr-2022 +###### Auto generated by spf13/cobra on 17-Apr-2022 diff --git a/docs/cmd/hrp_har2case.md b/docs/cmd/hrp_har2case.md index 5fd11b46..729b1226 100644 --- a/docs/cmd/hrp_har2case.md +++ b/docs/cmd/hrp_har2case.md @@ -24,4 +24,4 @@ hrp har2case $har_path... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 16-Apr-2022 +###### Auto generated by spf13/cobra on 17-Apr-2022 diff --git a/docs/cmd/hrp_pytest.md b/docs/cmd/hrp_pytest.md index 3f181b01..5a2363be 100644 --- a/docs/cmd/hrp_pytest.md +++ b/docs/cmd/hrp_pytest.md @@ -16,4 +16,4 @@ hrp pytest $path ... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 16-Apr-2022 +###### Auto generated by spf13/cobra on 17-Apr-2022 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index 3576c306..7ad8d4d4 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -34,4 +34,4 @@ hrp run $path... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 16-Apr-2022 +###### Auto generated by spf13/cobra on 17-Apr-2022 diff --git a/docs/cmd/hrp_startproject.md b/docs/cmd/hrp_startproject.md index d1c109bd..e33d46d6 100644 --- a/docs/cmd/hrp_startproject.md +++ b/docs/cmd/hrp_startproject.md @@ -19,4 +19,4 @@ hrp startproject $project_name [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 16-Apr-2022 +###### Auto generated by spf13/cobra on 17-Apr-2022 diff --git a/hrp/plugin.go b/hrp/plugin.go index a4beb671..a8144797 100644 --- a/hrp/plugin.go +++ b/hrp/plugin.go @@ -19,15 +19,17 @@ const ( hashicorpPyPluginFile = "debugtalk.py" // used for hashicorp python plugin ) -func initPlugin(path string, logOn bool) (plugin funplugin.IPlugin, err error) { +func initPlugin(path string, logOn bool) (plugin funplugin.IPlugin, pluginDir string, err error) { // plugin file not found if path == "" { - return nil, nil + return nil, "", nil } pluginPath, err := locatePlugin(path) if err != nil { - return nil, nil + return nil, "", nil } + // TODO: move pluginDir to funplugin + pluginDir = filepath.Dir(pluginPath) // found plugin file plugin, err = funplugin.Init(pluginPath, funplugin.WithLogOn(logOn)) diff --git a/hrp/runner.go b/hrp/runner.go index acdda315..fcc2cfe7 100644 --- a/hrp/runner.go +++ b/hrp/runner.go @@ -2,11 +2,9 @@ package hrp import ( "crypto/tls" - "fmt" "net" "net/http" "net/url" - "path/filepath" "testing" "time" @@ -15,7 +13,6 @@ import ( "github.com/rs/zerolog/log" "golang.org/x/net/http2" - "github.com/httprunner/httprunner/hrp/internal/builtin" "github.com/httprunner/httprunner/hrp/internal/sdk" ) @@ -175,22 +172,9 @@ func (r *HRPRunner) Run(testcases ...ITestCase) error { } s.Time.Duration = time.Since(s.Time.StartAt).Seconds() - // update the report output path - pluginPath, err := locatePlugin(testcases[0].GetPath()) - if err == nil { - outputPath, _ := filepath.Split(pluginPath) - summaryPath = filepath.Join(outputPath, summaryPath) - reportPath = filepath.Join(outputPath, reportPath) - } - // save summary if r.saveTests { - dir, _ := filepath.Split(summaryPath) - err := builtin.EnsureFolderExists(dir) - if err != nil { - return err - } - err = builtin.Dump2JSON(s, fmt.Sprintf(summaryPath, s.Time.StartAt.Unix())) + err := s.genSummary() if err != nil { return err } @@ -229,11 +213,12 @@ func (r *HRPRunner) newCaseRunner(testcase *TestCase) (*testCaseRunner, error) { } // init parser plugin - plugin, err := initPlugin(testcase.Config.Path, r.pluginLogOn) + plugin, pluginDir, err := initPlugin(testcase.Config.Path, r.pluginLogOn) if err != nil { return nil, errors.Wrap(err, "init plugin failed") } runner.parser.plugin = plugin + runner.rootDir = pluginDir // parse testcase config if err := runner.parseConfig(); err != nil { @@ -249,6 +234,7 @@ type testCaseRunner struct { parser *Parser parsedConfig *TConfig parametersIterator *ParametersIterator + rootDir string // project root dir } // parseConfig parses testcase config, stores to parsedConfig. diff --git a/hrp/summary.go b/hrp/summary.go index 569c79a6..42dc37d7 100644 --- a/hrp/summary.go +++ b/hrp/summary.go @@ -38,6 +38,7 @@ type Summary struct { Time *TestCaseTime `json:"time" yaml:"time"` Platform *Platform `json:"platform" yaml:"platform"` Details []*TestCaseSummary `json:"details" yaml:"details"` + rootDir string } func (s *Summary) appendCaseSummary(caseSummary *TestCaseSummary) { @@ -53,15 +54,24 @@ func (s *Summary) appendCaseSummary(caseSummary *TestCaseSummary) { s.Stat.TestSteps.Failures += caseSummary.Stat.Failures s.Details = append(s.Details, caseSummary) s.Success = s.Success && caseSummary.Success + + // specify output reports dir + if len(s.Details) == 1 { + s.rootDir = caseSummary.RootDir + } else if s.rootDir != caseSummary.RootDir { + // if multiple testcases have different root path, use current working dir + s.rootDir, _ = os.Getwd() + } } func (s *Summary) genHTMLReport() error { - dir, _ := filepath.Split(reportPath) - err := builtin.EnsureFolderExists(dir) + reportsDir := filepath.Join(s.rootDir, resultsDir) + err := builtin.EnsureFolderExists(reportsDir) if err != nil { return err } - reportPath := fmt.Sprintf(reportPath, s.Time.StartAt.Unix()) + + reportPath := filepath.Join(reportsDir, fmt.Sprintf("report-%v.html", s.Time.StartAt.Unix())) file, err := os.OpenFile(reportPath, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { log.Error().Err(err).Msg("open file failed") @@ -84,13 +94,25 @@ func (s *Summary) genHTMLReport() error { return err } +func (s *Summary) genSummary() error { + reportsDir := filepath.Join(s.rootDir, resultsDir) + err := builtin.EnsureFolderExists(reportsDir) + if err != nil { + return err + } + + summaryPath := filepath.Join(reportsDir, fmt.Sprintf("summary-%v.json", s.Time.StartAt.Unix())) + err = builtin.Dump2JSON(s, summaryPath) + if err != nil { + return err + } + return nil +} + //go:embed internal/scaffold/templates/report/template.html var reportTemplate string -var ( - reportPath = "reports/report-%v.html" - summaryPath = "reports/summary-%v.json" -) +const resultsDir = "reports" type Stat struct { TestCases TestCaseStat `json:"testcases" yaml:"test_cases"` @@ -130,6 +152,7 @@ type TestCaseSummary struct { InOut *TestCaseInOut `json:"in_out" yaml:"in_out"` Log string `json:"log,omitempty" yaml:"log,omitempty"` // TODO Records []*StepResult `json:"records" yaml:"records"` + RootDir string `json:"root_dir" yaml:"root_dir"` } type TestCaseInOut struct {