refactor: group parser functions

This commit is contained in:
debugtalk
2022-01-06 17:17:19 +08:00
parent 75ff0962c9
commit cfad6cee59
6 changed files with 179 additions and 169 deletions

113
parser.go
View File

@@ -4,8 +4,12 @@ import (
"encoding/json"
"fmt"
"net/url"
"os"
"path/filepath"
"plugin"
"reflect"
"regexp"
"runtime"
"strings"
"github.com/maja42/goval"
@@ -15,6 +19,87 @@ import (
"github.com/httprunner/hrp/internal/builtin"
)
func newParser() *parser {
return &parser{}
}
type parser struct {
// pluginLoader stores loaded go plugins.
pluginLoader *plugin.Plugin
}
func (p *parser) loadPlugin(path string) error {
if runtime.GOOS == "windows" {
log.Warn().Msg("go plugin does not support windows")
return nil
}
if path == "" {
return nil
}
// check if loaded before
if p.pluginLoader != nil {
return nil
}
// locate plugin file
pluginPath, err := locatePlugin(path)
if err != nil {
// plugin not found
return nil
}
// load plugin
plugins, err := plugin.Open(pluginPath)
if err != nil {
log.Error().Err(err).Str("path", path).Msg("load go plugin failed")
return err
}
p.pluginLoader = plugins
log.Info().Str("path", path).Msg("load go plugin success")
return nil
}
// locatePlugin searches debugtalk.so upward recursively until current
// working directory or system root dir.
func locatePlugin(startPath string) (string, error) {
stat, err := os.Stat(startPath)
if os.IsNotExist(err) {
return "", err
}
var startDir string
if stat.IsDir() {
startDir = startPath
} else {
startDir = filepath.Dir(startPath)
}
startDir, _ = filepath.Abs(startDir)
// convention over configuration
// target plugin file name is always debugtalk.so
pluginPath := filepath.Join(startDir, "debugtalk.so")
if _, err := os.Stat(pluginPath); err == nil {
return pluginPath, nil
}
// current working directory
cwd, _ := os.Getwd()
if startDir == cwd {
return "", fmt.Errorf("searched to CWD, plugin file not found")
}
// system root dir
parentDir, _ := filepath.Abs(filepath.Dir(startDir))
if parentDir == startDir {
return "", fmt.Errorf("searched to system root dir, plugin file not found")
}
return locatePlugin(parentDir)
}
func buildURL(baseURL, stepURL string) string {
uConfig, err := url.Parse(baseURL)
if err != nil {
@@ -32,9 +117,9 @@ func buildURL(baseURL, stepURL string) string {
return uStep.String()
}
func parseHeaders(rawHeaders map[string]string, variablesMapping map[string]interface{}, pluginLoader *pluginLoader) (map[string]string, error) {
func (p *parser) parseHeaders(rawHeaders map[string]string, variablesMapping map[string]interface{}) (map[string]string, error) {
parsedHeaders := make(map[string]string)
headers, err := parseData(rawHeaders, variablesMapping, pluginLoader)
headers, err := p.parseData(rawHeaders, variablesMapping)
if err != nil {
return rawHeaders, err
}
@@ -54,7 +139,7 @@ func convertString(raw interface{}) string {
}
}
func parseData(raw interface{}, variablesMapping map[string]interface{}, pluginLoader *pluginLoader) (interface{}, error) {
func (p *parser) parseData(raw interface{}, variablesMapping map[string]interface{}) (interface{}, error) {
rawValue := reflect.ValueOf(raw)
switch rawValue.Kind() {
case reflect.String:
@@ -65,11 +150,11 @@ func parseData(raw interface{}, variablesMapping map[string]interface{}, pluginL
// other string
value := rawValue.String()
value = strings.TrimSpace(value)
return parseString(value, variablesMapping, pluginLoader)
return p.parseString(value, variablesMapping)
case reflect.Slice:
parsedSlice := make([]interface{}, rawValue.Len())
for i := 0; i < rawValue.Len(); i++ {
parsedValue, err := parseData(rawValue.Index(i).Interface(), variablesMapping, pluginLoader)
parsedValue, err := p.parseData(rawValue.Index(i).Interface(), variablesMapping)
if err != nil {
return raw, err
}
@@ -79,12 +164,12 @@ func parseData(raw interface{}, variablesMapping map[string]interface{}, pluginL
case reflect.Map: // convert any map to map[string]interface{}
parsedMap := make(map[string]interface{})
for _, k := range rawValue.MapKeys() {
parsedKey, err := parseString(k.String(), variablesMapping, pluginLoader)
parsedKey, err := p.parseString(k.String(), variablesMapping)
if err != nil {
return raw, err
}
v := rawValue.MapIndex(k)
parsedValue, err := parseData(v.Interface(), variablesMapping, pluginLoader)
parsedValue, err := p.parseData(v.Interface(), variablesMapping)
if err != nil {
return raw, err
}
@@ -122,7 +207,7 @@ var (
)
// parseString parse string with variables
func parseString(raw string, variablesMapping map[string]interface{}, pluginLoader *pluginLoader) (interface{}, error) {
func (p *parser) parseString(raw string, variablesMapping map[string]interface{}) (interface{}, error) {
matchStartPosition := 0
parsedString := ""
remainedString := raw
@@ -161,12 +246,12 @@ func parseString(raw string, variablesMapping map[string]interface{}, pluginLoad
if err != nil {
return raw, err
}
parsedArgs, err := parseData(arguments, variablesMapping, pluginLoader)
parsedArgs, err := p.parseData(arguments, variablesMapping)
if err != nil {
return raw, err
}
fn, err := getMappingFunction(funcName, pluginLoader)
fn, err := getMappingFunction(funcName, p.pluginLoader)
if err != nil {
return raw, err
}
@@ -257,7 +342,7 @@ func mergeVariables(variables, overriddenVariables map[string]interface{}) map[s
return mergedVariables
}
func getMappingFunction(funcName string, pluginLoader *pluginLoader) (reflect.Value, error) {
func getMappingFunction(funcName string, pluginLoader *plugin.Plugin) (reflect.Value, error) {
var fn reflect.Value
var err error
@@ -384,7 +469,7 @@ func parseFunctionArguments(argsStr string) ([]interface{}, error) {
return arguments, nil
}
func parseVariables(variables map[string]interface{}, pluginLoader *pluginLoader) (map[string]interface{}, error) {
func (p *parser) parseVariables(variables map[string]interface{}) (map[string]interface{}, error) {
parsedVariables := make(map[string]interface{})
var traverseRounds int
@@ -422,7 +507,7 @@ func parseVariables(variables map[string]interface{}, pluginLoader *pluginLoader
return variables, fmt.Errorf("variable not defined: %v", undefinedVars)
}
parsedValue, err := parseData(varValue, parsedVariables, pluginLoader)
parsedValue, err := p.parseData(varValue, parsedVariables)
if err != nil {
continue
}
@@ -552,7 +637,7 @@ func parseParameters(parameters map[string]interface{}, variablesMapping map[str
case reflect.String:
// e.g. username-password: ${parameterize(examples/account.csv)} -> [{"username": "test1", "password": "111111"}, {"username": "test2", "password": "222222"}]
var parsedParameterContent interface{}
parsedParameterContent, err = parseString(rawValue.String(), variablesMapping, nil)
parsedParameterContent, err = newParser().parseString(rawValue.String(), variablesMapping)
if err != nil {
log.Error().Interface("parameterContent", rawValue).Msg("[parseParameters] parse parameter content error")
return nil, err

View File

@@ -1,6 +1,7 @@
package hrp
import (
"os"
"sort"
"testing"
"time"
@@ -8,6 +9,49 @@ import (
"github.com/stretchr/testify/assert"
)
func TestLocatePlugin(t *testing.T) {
cwd, _ := os.Getwd()
_, err := locatePlugin(cwd)
if !assert.Error(t, err) {
t.Fail()
}
_, err = locatePlugin("")
if !assert.Error(t, err) {
t.Fail()
}
startPath := "examples/debugtalk.so"
_, err = locatePlugin(startPath)
if !assert.Nil(t, err) {
t.Fail()
}
startPath = "examples/demo.json"
_, err = locatePlugin(startPath)
if !assert.Nil(t, err) {
t.Fail()
}
startPath = "examples/"
_, err = locatePlugin(startPath)
if !assert.Nil(t, err) {
t.Fail()
}
startPath = "examples/plugin/debugtalk.go"
_, err = locatePlugin(startPath)
if !assert.Nil(t, err) {
t.Fail()
}
startPath = "/abc"
_, err = locatePlugin(startPath)
if !assert.Error(t, err) {
t.Fail()
}
}
func TestBuildURL(t *testing.T) {
var url string
url = buildURL("https://postman-echo.com", "/get")
@@ -160,8 +204,9 @@ func TestParseDataStringWithVariables(t *testing.T) {
{"abc$var_5", "abctrue"}, // "abcTrue"
}
parser := newParser()
for _, data := range testData {
parsedData, err := parseData(data.expr, variablesMapping, nil)
parsedData, err := parser.parseData(data.expr, variablesMapping)
if !assert.NoError(t, err) {
t.Fail()
}
@@ -184,8 +229,9 @@ func TestParseDataStringWithUndefinedVariables(t *testing.T) {
{"/api/$SECRET_KEY", "/api/$SECRET_KEY"}, // raise error
}
parser := newParser()
for _, data := range testData {
parsedData, err := parseData(data.expr, variablesMapping, nil)
parsedData, err := parser.parseData(data.expr, variablesMapping)
if !assert.Error(t, err) {
t.Fail()
}
@@ -228,8 +274,9 @@ func TestParseDataStringWithVariablesAbnormal(t *testing.T) {
{"ABC$var_1{}a", "ABCabc{}a"}, // {}
}
parser := newParser()
for _, data := range testData {
parsedData, err := parseData(data.expr, variablesMapping, nil)
parsedData, err := parser.parseData(data.expr, variablesMapping)
if !assert.NoError(t, err) {
t.Fail()
}
@@ -258,8 +305,9 @@ func TestParseDataMapWithVariables(t *testing.T) {
{map[string]interface{}{"$var2": "$val1"}, map[string]interface{}{"123": 200}},
}
parser := newParser()
for _, data := range testData {
parsedData, err := parseData(data.expr, variablesMapping, nil)
parsedData, err := parser.parseData(data.expr, variablesMapping)
if !assert.NoError(t, err) {
t.Fail()
}
@@ -291,8 +339,9 @@ func TestParseHeaders(t *testing.T) {
{map[string]string{"$var2": "$val2"}, map[string]string{"123": "<nil>"}},
}
parser := newParser()
for _, data := range testData {
parsedHeaders, err := parseHeaders(data.rawHeaders, variablesMapping, nil)
parsedHeaders, err := parser.parseHeaders(data.rawHeaders, variablesMapping)
if !assert.NoError(t, err) {
t.Fail()
}
@@ -444,8 +493,9 @@ func TestParseDataStringWithFunctions(t *testing.T) {
{"123${gen_random_string($n)}abc", 11},
}
parser := newParser()
for _, data := range testData1 {
value, err := parseData(data.expr, variablesMapping, nil)
value, err := parser.parseData(data.expr, variablesMapping)
if !assert.NoError(t, err) {
t.Fail()
}
@@ -464,7 +514,7 @@ func TestParseDataStringWithFunctions(t *testing.T) {
}
for _, data := range testData2 {
value, err := parseData(data.expr, variablesMapping, nil)
value, err := parser.parseData(data.expr, variablesMapping)
if !assert.NoError(t, err) {
t.Fail()
}
@@ -510,8 +560,9 @@ func TestParseVariables(t *testing.T) {
},
}
parser := newParser()
for _, data := range testData {
value, err := parseVariables(data.rawVars, nil)
value, err := parser.parseVariables(data.rawVars)
if !assert.NoError(t, err) {
t.Fail()
}
@@ -540,8 +591,9 @@ func TestParseVariablesAbnormal(t *testing.T) {
},
}
parser := newParser()
for _, data := range testData {
value, err := parseVariables(data.rawVars, nil)
value, err := parser.parseVariables(data.rawVars)
if !assert.Error(t, err) {
t.Fail()
}

View File

@@ -27,11 +27,10 @@ func TestMain(m *testing.M) {
}
func TestCallPluginFunction(t *testing.T) {
plugins, err := plugin.Open("examples/debugtalk.so")
pluginLoader, err := plugin.Open("examples/debugtalk.so")
if err != nil {
t.Fatalf(err.Error())
}
pluginLoader := &pluginLoader{plugins}
// call function without arguments
f1, _ := getMappingFunction("Concatenate", pluginLoader)

View File

@@ -13,7 +13,7 @@ import (
"github.com/httprunner/hrp/internal/builtin"
)
func newResponseObject(t *testing.T, pluginLoader *pluginLoader, resp *http.Response) (*responseObject, error) {
func newResponseObject(t *testing.T, parser *parser, resp *http.Response) (*responseObject, error) {
// prepare response headers
headers := make(map[string]string)
for k, v := range resp.Header {
@@ -60,9 +60,9 @@ func newResponseObject(t *testing.T, pluginLoader *pluginLoader, resp *http.Resp
}
return &responseObject{
t: t,
pluginLoader: pluginLoader,
respObjMeta: data,
t: t,
parser: parser,
respObjMeta: data,
}, nil
}
@@ -75,7 +75,7 @@ type respObjMeta struct {
type responseObject struct {
t *testing.T
pluginLoader *pluginLoader
parser *parser
respObjMeta interface{}
validationResults map[string]interface{}
}
@@ -103,7 +103,7 @@ func (v *responseObject) Validate(validators []Validator, variablesMapping map[s
var checkValue interface{}
if strings.Contains(checkItem, "$") {
// reference variable
checkValue, err = parseData(checkItem, variablesMapping, v.pluginLoader)
checkValue, err = v.parser.parseData(checkItem, variablesMapping)
if err != nil {
return err
}
@@ -116,7 +116,7 @@ func (v *responseObject) Validate(validators []Validator, variablesMapping map[s
assertFunc := builtin.Assertions[assertMethod]
// parse expected value
expectValue, err := parseData(validator.Expect, variablesMapping, v.pluginLoader)
expectValue, err := v.parser.parseData(validator.Expect, variablesMapping)
if err != nil {
return err
}

102
runner.go
View File

@@ -9,10 +9,6 @@ import (
"net/http"
"net/http/httputil"
"net/url"
"os"
"path/filepath"
"plugin"
"runtime"
"strconv"
"strings"
"testing"
@@ -130,6 +126,7 @@ func (r *HRPRunner) newCaseRunner(testcase *TestCase) *caseRunner {
caseRunner := &caseRunner{
TestCase: testcase,
hrpRunner: r,
parser: newParser(),
}
caseRunner.reset()
return caseRunner
@@ -140,13 +137,12 @@ func (r *HRPRunner) newCaseRunner(testcase *TestCase) *caseRunner {
type caseRunner struct {
*TestCase
hrpRunner *HRPRunner
parser *parser
sessionVariables map[string]interface{}
// transactions stores transaction timing info.
// key is transaction name, value is map of transaction type and time, e.g. start time and end time.
transactions map[string]map[transactionType]time.Time
startTime time.Time // record start time of the testcase
// pluginLoader stores loaded go plugins.
pluginLoader *pluginLoader
}
// reset clears runner session variables.
@@ -209,7 +205,7 @@ func (r *caseRunner) runStep(index int, caseConfig *TConfig) (stepResult *stepDa
stepVariables = mergeVariables(stepVariables, caseConfig.Variables)
// parse step variables
parsedVariables, err := parseVariables(stepVariables, r.pluginLoader)
parsedVariables, err := r.parser.parseVariables(stepVariables)
if err != nil {
log.Error().Interface("variables", caseConfig.Variables).Err(err).Msg("parse step variables failed")
return nil, err
@@ -326,7 +322,7 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro
// prepare request headers
if len(step.Request.Headers) > 0 {
headers, err := parseHeaders(step.Request.Headers, step.Variables, r.pluginLoader)
headers, err := r.parser.parseHeaders(step.Request.Headers, step.Variables)
if err != nil {
return nil, errors.Wrap(err, "parse headers failed")
}
@@ -343,7 +339,7 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro
// prepare request params
var queryParams url.Values
if len(step.Request.Params) > 0 {
params, err := parseData(step.Request.Params, step.Variables, r.pluginLoader)
params, err := r.parser.parseData(step.Request.Params, step.Variables)
if err != nil {
return nil, errors.Wrap(err, "parse data failed")
}
@@ -375,7 +371,7 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro
// prepare request body
if step.Request.Body != nil {
data, err := parseData(step.Request.Body, step.Variables, r.pluginLoader)
data, err := r.parser.parseData(step.Request.Body, step.Variables)
if err != nil {
return nil, err
}
@@ -449,7 +445,7 @@ func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err erro
}
// new response object
respObj, err := newResponseObject(r.hrpRunner.t, r.pluginLoader, resp)
respObj, err := newResponseObject(r.hrpRunner.t, r.parser, resp)
if err != nil {
err = errors.Wrap(err, "init ResponseObject error")
return
@@ -494,7 +490,7 @@ func (r *caseRunner) runStepTestCase(step *TStep) (stepResult *stepData, err err
func (r *caseRunner) parseConfig(config IConfig) error {
cfg := config.ToStruct()
// parse config variables
parsedVariables, err := parseVariables(cfg.Variables, r.pluginLoader)
parsedVariables, err := r.parser.parseVariables(cfg.Variables)
if err != nil {
log.Error().Interface("variables", cfg.Variables).Err(err).Msg("parse config variables failed")
return err
@@ -502,20 +498,20 @@ func (r *caseRunner) parseConfig(config IConfig) error {
cfg.Variables = parsedVariables
// load plugin variables and functions
err = r.loadPlugin(cfg.Path)
err = r.parser.loadPlugin(cfg.Path)
if err != nil {
return err
}
// parse config name
parsedName, err := parseString(cfg.Name, cfg.Variables, r.pluginLoader)
parsedName, err := r.parser.parseString(cfg.Name, cfg.Variables)
if err != nil {
return err
}
cfg.Name = convertString(parsedName)
// parse config base url
parsedBaseURL, err := parseString(cfg.BaseURL, cfg.Variables, r.pluginLoader)
parsedBaseURL, err := r.parser.parseString(cfg.BaseURL, cfg.Variables)
if err != nil {
return err
}
@@ -524,86 +520,10 @@ func (r *caseRunner) parseConfig(config IConfig) error {
return nil
}
type pluginLoader struct {
*plugin.Plugin
}
func (r *caseRunner) loadPlugin(path string) error {
if runtime.GOOS == "windows" {
log.Warn().Msg("go plugin does not support windows")
return nil
}
if path == "" {
return nil
}
// check if loaded before
if r.pluginLoader != nil {
return nil
}
// locate plugin file
pluginPath, err := locatePlugin(path)
if err != nil {
// plugin not found
return nil
}
// load plugin
plugins, err := plugin.Open(pluginPath)
if err != nil {
log.Error().Err(err).Str("path", path).Msg("load go plugin failed")
return err
}
r.pluginLoader = &pluginLoader{plugins}
log.Info().Str("path", path).Msg("load go plugin success")
return nil
}
func (r *caseRunner) getSummary() *testCaseSummary {
return &testCaseSummary{}
}
// locatePlugin searches debugtalk.so upward recursively until current
// working directory or system root dir.
func locatePlugin(startPath string) (string, error) {
stat, err := os.Stat(startPath)
if os.IsNotExist(err) {
return "", err
}
var startDir string
if stat.IsDir() {
startDir = startPath
} else {
startDir = filepath.Dir(startPath)
}
startDir, _ = filepath.Abs(startDir)
// convention over configuration
// target plugin file name is always debugtalk.so
pluginPath := filepath.Join(startDir, "debugtalk.so")
if _, err := os.Stat(pluginPath); err == nil {
return pluginPath, nil
}
// current working directory
cwd, _ := os.Getwd()
if startDir == cwd {
return "", fmt.Errorf("searched to CWD, plugin file not found")
}
// system root dir
parentDir, _ := filepath.Abs(filepath.Dir(startDir))
if parentDir == startDir {
return "", fmt.Errorf("searched to system root dir, plugin file not found")
}
return locatePlugin(parentDir)
}
func setBodyBytes(req *http.Request, data []byte) {
req.Body = ioutil.NopCloser(bytes.NewReader(data))
req.ContentLength = int64(len(data))

View File

@@ -1,55 +1,9 @@
package hrp
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLocatePlugin(t *testing.T) {
cwd, _ := os.Getwd()
_, err := locatePlugin(cwd)
if !assert.Error(t, err) {
t.Fail()
}
_, err = locatePlugin("")
if !assert.Error(t, err) {
t.Fail()
}
startPath := "examples/debugtalk.so"
_, err = locatePlugin(startPath)
if !assert.Nil(t, err) {
t.Fail()
}
startPath = "examples/demo.json"
_, err = locatePlugin(startPath)
if !assert.Nil(t, err) {
t.Fail()
}
startPath = "examples/"
_, err = locatePlugin(startPath)
if !assert.Nil(t, err) {
t.Fail()
}
startPath = "examples/plugin/debugtalk.go"
_, err = locatePlugin(startPath)
if !assert.Nil(t, err) {
t.Fail()
}
startPath = "/abc"
_, err = locatePlugin(startPath)
if !assert.Error(t, err) {
t.Fail()
}
}
func TestHttpRunner(t *testing.T) {
testcase1 := &TestCase{
Config: NewConfig("TestCase1").