From 2d105248fbe9ea0c5a0dc4d7f53dd9acef7611c0 Mon Sep 17 00:00:00 2001 From: buyuxiang <347586493@qq.com> Date: Fri, 24 Jun 2022 17:46:22 +0800 Subject: [PATCH 1/2] support variable reference during extraction --- docs/CHANGELOG.md | 4 +++- hrp/response.go | 41 +++++++++++++++++++-------------------- hrp/step_request.go | 2 +- hrp/step_websocket.go | 2 +- hrp/tests/extract_test.go | 3 ++- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0bda2b7d..62ff1977 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## v4.1.5 (2022-06-26) +## v4.1.5 (2022-06-27) **go version** @@ -12,6 +12,8 @@ - fix: failed to load json/data content in api reference - fix: failed to convert postman collection containing multipart/form-data requests to pytest - fix: only get the first parameter in referenced testcase +- fix: support variable reference during extraction +- fix: simplify jmespath compatibility conversion **python version** diff --git a/hrp/response.go b/hrp/response.go index 33b363cb..43c2d80b 100644 --- a/hrp/response.go +++ b/hrp/response.go @@ -123,24 +123,35 @@ type responseObject struct { const textExtractorSubRegexp string = `(.*)` -func (v *responseObject) searchField(value string) interface{} { - var result interface{} - if strings.Contains(value, textExtractorSubRegexp) { - result = v.searchRegexp(value) - } else { - result = v.searchJmespath(value) +func (v *responseObject) searchField(field string, variablesMapping map[string]interface{}) interface{} { + var result interface{} = field + if strings.Contains(field, "$") { + // parse reference variables in field before search + var err error + result, err = v.parser.Parse(field, variablesMapping) + if err != nil { + log.Error().Str("filed name", field).Err(err).Msg("fail to parse field before search") + } + } + // search field using jmespath or regex if parsed field is still string and contains specified fieldTags + if parsedField, ok := result.(string); ok && checkSearchField(parsedField) { + if strings.Contains(field, textExtractorSubRegexp) { + result = v.searchRegexp(parsedField) + } else { + result = v.searchJmespath(parsedField) + } } return result } -func (v *responseObject) Extract(extractors map[string]string) map[string]interface{} { +func (v *responseObject) Extract(extractors map[string]string, variablesMapping map[string]interface{}) map[string]interface{} { if extractors == nil { return nil } extractMapping := make(map[string]interface{}) for key, value := range extractors { - extractedValue := v.searchField(value) + extractedValue := v.searchField(value, variablesMapping) log.Info().Str("from", value).Interface("value", extractedValue).Msg("extract value") log.Info().Str("variable", key).Interface("value", extractedValue).Msg("set variable") extractMapping[key] = extractedValue @@ -157,19 +168,7 @@ func (v *responseObject) Validate(iValidators []interface{}, variablesMapping ma } // parse check value checkItem := validator.Check - var checkValue interface{} - if strings.Contains(checkItem, "$") { - // parse reference variables in checkItem - checkValue, err = v.parser.Parse(checkItem, variablesMapping) - if err != nil { - return err - } - } else { - checkValue = checkItem - } - if searchCheckValue, ok := checkValue.(string); ok && checkSearchField(searchCheckValue) { - checkValue = v.searchField(searchCheckValue) - } + checkValue := v.searchField(checkItem, variablesMapping) // get assert method assertMethod := validator.Assert diff --git a/hrp/step_request.go b/hrp/step_request.go index d2ce333a..29a86578 100644 --- a/hrp/step_request.go +++ b/hrp/step_request.go @@ -419,7 +419,7 @@ func runStepRequest(r *SessionRunner, step *TStep) (stepResult *StepResult, err // extract variables from response extractors := step.Extract - extractMapping := respObj.Extract(extractors) + extractMapping := respObj.Extract(extractors, stepVariables) stepResult.ExportVars = extractMapping // override step variables with extracted variables diff --git a/hrp/step_websocket.go b/hrp/step_websocket.go index f3a788fd..34cc226c 100644 --- a/hrp/step_websocket.go +++ b/hrp/step_websocket.go @@ -365,7 +365,7 @@ func runStepWebSocket(r *SessionRunner, step *TStep) (stepResult *StepResult, er // extract variables from response extractors := step.Extract - extractMapping := respObj.Extract(extractors) + extractMapping := respObj.Extract(extractors, stepVariables) stepResult.ExportVars = extractMapping // override step variables with extracted variables diff --git a/hrp/tests/extract_test.go b/hrp/tests/extract_test.go index df8d2d18..9c2a8848 100644 --- a/hrp/tests/extract_test.go +++ b/hrp/tests/extract_test.go @@ -18,6 +18,7 @@ func TestCaseExtractStepSingle(t *testing.T) { "var1": "bar1", "agent": "HttpRunnerPlus", "expectedStatusCode": 200, + "jmespathFoo1": "body.args.foo1", }). GET("/get"). WithParams(map[string]interface{}{"foo1": "$var1", "foo2": "bar2"}). @@ -25,7 +26,7 @@ func TestCaseExtractStepSingle(t *testing.T) { Extract(). WithJmesPath("status_code", "statusCode"). WithJmesPath("headers.\"Content-Type\"", "contentType"). - WithJmesPath("body.args.foo1", "varFoo1"). + WithJmesPath("$jmespathFoo1", "varFoo1"). Validate(). AssertEqual("$statusCode", "$expectedStatusCode", "check status code"). // assert with extracted variable from current step AssertEqual("$contentType", "application/json; charset=utf-8", "check header Content-Type"). // assert with extracted variable from current step From 3f8aa2dc0374b827ce9282cdaaa4415576fd31a3 Mon Sep 17 00:00:00 2001 From: buyuxiang <347586493@qq.com> Date: Mon, 27 Jun 2022 18:08:23 +0800 Subject: [PATCH 2/2] simplify jmespath compatibility conversion --- .../templates/testcases/demo_ref_api.json | 2 +- hrp/testcase.go | 17 ++++--- hrp/testcase_test.go | 15 +++--- httprunner/compat.py | 21 ++------ httprunner/compat_test.py | 50 ++++++------------- 5 files changed, 39 insertions(+), 66 deletions(-) diff --git a/hrp/internal/scaffold/templates/testcases/demo_ref_api.json b/hrp/internal/scaffold/templates/testcases/demo_ref_api.json index 7bc33c5e..4b17e373 100644 --- a/hrp/internal/scaffold/templates/testcases/demo_ref_api.json +++ b/hrp/internal/scaffold/templates/testcases/demo_ref_api.json @@ -52,7 +52,7 @@ "msg": "check status_code" }, { - "check": "body.headers.postman-token", + "check": "body.headers.\"postman-token\"", "assert": "equal", "expect": "ea19464c-ddd4-4724-abe9-5e2b254c2723", "msg": "check body.headers.postman-token" diff --git a/hrp/testcase.go b/hrp/testcase.go index bae01478..6a91939b 100644 --- a/hrp/testcase.go +++ b/hrp/testcase.go @@ -199,6 +199,9 @@ func (tc *TCase) MakeCompat() (err error) { func convertCompatRequestBody(request *Request) { if request != nil && request.Body == nil { if request.Json != nil { + if request.Headers == nil { + request.Headers = make(map[string]string) + } request.Headers["Content-Type"] = "application/json; charset=utf-8" request.Body = request.Json request.Json = nil @@ -228,7 +231,7 @@ func convertCompatValidator(Validators []interface{}) (err error) { if iMsg, msgExisted := validatorMap["msg"]; msgExisted { validator.Message = iMsg.(string) } - validator.Check = convertCheckExpr(validator.Check) + validator.Check = convertJmespathExpr(validator.Check) Validators[i] = validator continue } @@ -246,7 +249,7 @@ func convertCompatValidator(Validators []interface{}) (err error) { validator.Message = validatorContent[2].(string) } } - validator.Check = convertCheckExpr(validator.Check) + validator.Check = convertJmespathExpr(validator.Check) Validators[i] = validator continue } @@ -258,18 +261,20 @@ func convertCompatValidator(Validators []interface{}) (err error) { // convertExtract deals with extract expr including hyphen func convertExtract(extract map[string]string) { for key, value := range extract { - extract[key] = convertCheckExpr(value) + extract[key] = convertJmespathExpr(value) } } -// convertCheckExpr deals with check expression including hyphen -func convertCheckExpr(checkExpr string) string { +// convertJmespathExpr deals with limited jmespath expression conversion +func convertJmespathExpr(checkExpr string) string { if strings.Contains(checkExpr, textExtractorSubRegexp) { return checkExpr } checkItems := strings.Split(checkExpr, ".") for i, checkItem := range checkItems { - if strings.Contains(checkItem, "-") && !strings.Contains(checkItem, "\"") { + checkItem = strings.Trim(checkItem, "\"") + lowerItem := strings.ToLower(checkItem) + if strings.HasPrefix(lowerItem, "content-") || lowerItem == "user-agent" { checkItems[i] = fmt.Sprintf("\"%s\"", checkItem) } } diff --git a/hrp/testcase_test.go b/hrp/testcase_test.go index c55642e1..13417460 100644 --- a/hrp/testcase_test.go +++ b/hrp/testcase_test.go @@ -210,21 +210,20 @@ func TestConvertCheckExpr(t *testing.T) { }{ // normal check expression {"a.b.c", "a.b.c"}, - {"headers.\"Content-Type\"", "headers.\"Content-Type\""}, + {"a.\"b-c\".d", "a.\"b-c\".d"}, + {"a.b-c.d", "a.b-c.d"}, + {"body.args.a[-1]", "body.args.a[-1]"}, // check expression using regex {"covering (.*) testing,", "covering (.*) testing,"}, {" (.*) a-b-c", " (.*) a-b-c"}, // abnormal check expression - {"-", "\"-\""}, - {"b-c", "\"b-c\""}, - {"a.b-c.d", "a.\"b-c\".d"}, - {"a-b.c-d", "\"a-b\".\"c-d\""}, - {"\"a-b\".c-d", "\"a-b\".\"c-d\""}, {"headers.Content-Type", "headers.\"Content-Type\""}, - {"body.I-am-a-Key.name", "body.\"I-am-a-Key\".name"}, + {"headers.\"Content-Type", "headers.\"Content-Type\""}, + {"headers.Content-Type\"", "headers.\"Content-Type\""}, + {"headers.User-Agent", "headers.\"User-Agent\""}, } for _, expr := range exprs { - if !assert.Equal(t, convertCheckExpr(expr.before), expr.after) { + if !assert.Equal(t, expr.after, convertJmespathExpr(expr.before)) { t.Fatal() } } diff --git a/httprunner/compat.py b/httprunner/compat.py index 4732783a..755d600e 100644 --- a/httprunner/compat.py +++ b/httprunner/compat.py @@ -14,7 +14,7 @@ from httprunner.utils import sort_dict_by_custom_order def convert_variables( - raw_variables: Union[Dict, Text], test_path: Text + raw_variables: Union[Dict, Text], test_path: Text ) -> Dict[Text, Any]: if isinstance(raw_variables, Dict): return raw_variables @@ -54,24 +54,13 @@ def _convert_jmespath(raw: Text) -> Text: elif raw.startswith("json"): raw = f"body{raw[len('json'):]}" - raw_list = [] - for item in raw.split("."): + raw_list = raw.split(".") + for i, item in enumerate(raw_list): + item = item.strip('"') if item.lower().startswith("content-") or item.lower() in ["user-agent"]: # add quotes for some field in white list # e.g. headers.Content-Type => headers."Content-Type" - item = item.strip('"') - raw_list.append(f'"{item}"') - elif item.isdigit(): - # convert lst.0.name to lst[0].name - if len(raw_list) == 0: - logger.error(f"Invalid jmespath: {raw}") - sys.exit(1) - - last_item = raw_list.pop() - item = f"{last_item}[{item}]" - raw_list.append(item) - else: - raw_list.append(item) + raw_list[i] = f'"{item}"' return ".".join(raw_list) diff --git a/httprunner/compat_test.py b/httprunner/compat_test.py index a7877fb6..6ae50a51 100644 --- a/httprunner/compat_test.py +++ b/httprunner/compat_test.py @@ -30,47 +30,33 @@ class TestCompat(unittest.TestCase): request_with_json_body = { "method": "POST", "url": "https://postman-echo.com/post", - "headers": { - "Content-Type": "application/json" - }, - "body": { - "k1": "v1", - "k2": "v2" - } + "headers": {"Content-Type": "application/json"}, + "body": {"k1": "v1", "k2": "v2"}, } self.assertEqual( compat._convert_request(request_with_json_body), { "method": "POST", "url": "https://postman-echo.com/post", - "headers": { - "Content-Type": "application/json" - }, - "json": { - "k1": "v1", - "k2": "v2" - } - } + "headers": {"Content-Type": "application/json"}, + "json": {"k1": "v1", "k2": "v2"}, + }, ) request_with_text_body = { "method": "POST", "url": "https://postman-echo.com/post", - "headers": { - "Content-Type": "text/plain" - }, - "body": "have a nice day" + "headers": {"Content-Type": "text/plain"}, + "body": "have a nice day", } self.assertEqual( compat._convert_request(request_with_text_body), { "method": "POST", "url": "https://postman-echo.com/post", - "headers": { - "Content-Type": "text/plain" - }, - "data": "have a nice day" - } + "headers": {"Content-Type": "text/plain"}, + "data": "have a nice day", + }, ) def test_convert_jmespath(self): @@ -85,10 +71,6 @@ class TestCompat(unittest.TestCase): self.assertEqual( compat._convert_jmespath('headers."Content-Type"'), 'headers."Content-Type"' ) - self.assertEqual( - compat._convert_jmespath("body.data.buildings.0.building_id"), - "body.data.buildings[0].building_id", - ) self.assertEqual( compat._convert_jmespath("body.users[-1]"), "body.users[-1]", @@ -97,8 +79,6 @@ class TestCompat(unittest.TestCase): compat._convert_jmespath("body.result.WorkNode_-1"), "body.result.WorkNode_-1", ) - with self.assertRaises(SystemExit): - compat._convert_jmespath("2.buildings.0.building_id") def test_convert_extractors(self): self.assertEqual( @@ -108,11 +88,11 @@ class TestCompat(unittest.TestCase): {"varA": "body.varA", "varB": "body.varB"}, ) self.assertEqual( - compat._convert_extractors([{"varA": "content.0.varA"}]), + compat._convert_extractors([{"varA": "content[0].varA"}]), {"varA": "body[0].varA"}, ) self.assertEqual( - compat._convert_extractors({"varA": "content.0.varA"}), + compat._convert_extractors({"varA": "content[0].varA"}), {"varA": "body[0].varA"}, ) @@ -128,7 +108,7 @@ class TestCompat(unittest.TestCase): [{"eq": ["body.abc", 201]}], ) self.assertEqual( - compat._convert_validators([{"eq": ["content.0.name", 201]}]), + compat._convert_validators([{"eq": ["content[0].name", 201]}]), [{"eq": ["body[0].name", 201]}], ) @@ -142,7 +122,7 @@ class TestCompat(unittest.TestCase): "headers": {"User-Agent": "HttpRunner/3.0"}, }, "extract": [{"varA": "content.varA"}, {"user_agent": "headers.User-Agent"}], - "validate": [{"eq": ["content.varB", 200]}, {"lt": ["json.0.varC", 0]}], + "validate": [{"eq": ["content.varB", 200]}, {"lt": ["json[0].varC", 0]}], } self.assertEqual( compat.ensure_testcase_v4_api(api_content), @@ -191,7 +171,7 @@ class TestCompat(unittest.TestCase): ], "validate": [ {"eq": ["content.varB", 200]}, - {"lt": ["json.0.varC", 0]}, + {"lt": ["json[0].varC", 0]}, ], } ],