Merge pull request #1386 from bbx-winner/fix-extract

fix: support variable reference during extraction
2026-05-12 02:21:29 +08:00 · 2022-06-27 18:41:14 +08:00
parent 2caf5d5159 3f8aa2dc03
commit 3e7d8692cb
10 changed files with 66 additions and 91 deletions
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,6 @@
 # Release History

-## v4.1.5 (2022-06-26)
+## v4.1.5 (2022-06-27)

 **go version**

@@ -12,6 +12,8 @@
 - fix: failed to load json/data content in api reference
 - fix: failed to convert postman collection containing multipart/form-data requests to pytest
 - fix: only get the first parameter in referenced testcase
+- fix: support variable reference during extraction
+- fix: simplify jmespath compatibility conversion

 **python version**

--- a/hrp/internal/scaffold/templates/testcases/demo_ref_api.json
+++ b/hrp/internal/scaffold/templates/testcases/demo_ref_api.json
@@ -52,7 +52,7 @@
                    "msg": "check status_code"
                },
                {
-                    "check": "body.headers.postman-token",
+                    "check": "body.headers.\"postman-token\"",
                    "assert": "equal",
                    "expect": "ea19464c-ddd4-4724-abe9-5e2b254c2723",
                    "msg": "check body.headers.postman-token"
--- a/hrp/response.go
+++ b/hrp/response.go
@@ -123,24 +123,35 @@ type responseObject struct {

 const textExtractorSubRegexp string = `(.*)`

-func (v *responseObject) searchField(value string) interface{} {
-	var result interface{}
-	if strings.Contains(value, textExtractorSubRegexp) {
-		result = v.searchRegexp(value)
-	} else {
-		result = v.searchJmespath(value)
+func (v *responseObject) searchField(field string, variablesMapping map[string]interface{}) interface{} {
+	var result interface{} = field
+	if strings.Contains(field, "$") {
+		// parse reference variables in field before search
+		var err error
+		result, err = v.parser.Parse(field, variablesMapping)
+		if err != nil {
+			log.Error().Str("filed name", field).Err(err).Msg("fail to parse field before search")
+		}
+	}
+	// search field using jmespath or regex if parsed field is still string and contains specified fieldTags
+	if parsedField, ok := result.(string); ok && checkSearchField(parsedField) {
+		if strings.Contains(field, textExtractorSubRegexp) {
+			result = v.searchRegexp(parsedField)
+		} else {
+			result = v.searchJmespath(parsedField)
+		}
 	}
 	return result
 }

-func (v *responseObject) Extract(extractors map[string]string) map[string]interface{} {
+func (v *responseObject) Extract(extractors map[string]string, variablesMapping map[string]interface{}) map[string]interface{} {
 	if extractors == nil {
 		return nil
 	}

 	extractMapping := make(map[string]interface{})
 	for key, value := range extractors {
-		extractedValue := v.searchField(value)
+		extractedValue := v.searchField(value, variablesMapping)
 		log.Info().Str("from", value).Interface("value", extractedValue).Msg("extract value")
 		log.Info().Str("variable", key).Interface("value", extractedValue).Msg("set variable")
 		extractMapping[key] = extractedValue
@@ -157,19 +168,7 @@ func (v *responseObject) Validate(iValidators []interface{}, variablesMapping ma
 		}
 		// parse check value
 		checkItem := validator.Check
-		var checkValue interface{}
-		if strings.Contains(checkItem, "$") {
-			// parse reference variables in checkItem
-			checkValue, err = v.parser.Parse(checkItem, variablesMapping)
-			if err != nil {
-				return err
-			}
-		} else {
-			checkValue = checkItem
-		}
-		if searchCheckValue, ok := checkValue.(string); ok && checkSearchField(searchCheckValue) {
-			checkValue = v.searchField(searchCheckValue)
-		}
+		checkValue := v.searchField(checkItem, variablesMapping)

 		// get assert method
 		assertMethod := validator.Assert
--- a/hrp/step_request.go
+++ b/hrp/step_request.go
@@ -419,7 +419,7 @@ func runStepRequest(r *SessionRunner, step *TStep) (stepResult *StepResult, err

 	// extract variables from response
 	extractors := step.Extract
-	extractMapping := respObj.Extract(extractors)
+	extractMapping := respObj.Extract(extractors, stepVariables)
 	stepResult.ExportVars = extractMapping

 	// override step variables with extracted variables
--- a/hrp/step_websocket.go
+++ b/hrp/step_websocket.go
@@ -365,7 +365,7 @@ func runStepWebSocket(r *SessionRunner, step *TStep) (stepResult *StepResult, er

 		// extract variables from response
 		extractors := step.Extract
-		extractMapping := respObj.Extract(extractors)
+		extractMapping := respObj.Extract(extractors, stepVariables)
 		stepResult.ExportVars = extractMapping

 		// override step variables with extracted variables
--- a/hrp/testcase.go
+++ b/hrp/testcase.go
@@ -199,6 +199,9 @@ func (tc *TCase) MakeCompat() (err error) {
 func convertCompatRequestBody(request *Request) {
 	if request != nil && request.Body == nil {
 		if request.Json != nil {
+			if request.Headers == nil {
+				request.Headers = make(map[string]string)
+			}
 			request.Headers["Content-Type"] = "application/json; charset=utf-8"
 			request.Body = request.Json
 			request.Json = nil
@@ -228,7 +231,7 @@ func convertCompatValidator(Validators []interface{}) (err error) {
 			if iMsg, msgExisted := validatorMap["msg"]; msgExisted {
 				validator.Message = iMsg.(string)
 			}
-			validator.Check = convertCheckExpr(validator.Check)
+			validator.Check = convertJmespathExpr(validator.Check)
 			Validators[i] = validator
 			continue
 		}
@@ -246,7 +249,7 @@ func convertCompatValidator(Validators []interface{}) (err error) {
 					validator.Message = validatorContent[2].(string)
 				}
 			}
-			validator.Check = convertCheckExpr(validator.Check)
+			validator.Check = convertJmespathExpr(validator.Check)
 			Validators[i] = validator
 			continue
 		}
@@ -258,18 +261,20 @@ func convertCompatValidator(Validators []interface{}) (err error) {
 // convertExtract deals with extract expr including hyphen
 func convertExtract(extract map[string]string) {
 	for key, value := range extract {
-		extract[key] = convertCheckExpr(value)
+		extract[key] = convertJmespathExpr(value)
 	}
 }

-// convertCheckExpr deals with check expression including hyphen
-func convertCheckExpr(checkExpr string) string {
+// convertJmespathExpr deals with limited jmespath expression conversion
+func convertJmespathExpr(checkExpr string) string {
 	if strings.Contains(checkExpr, textExtractorSubRegexp) {
 		return checkExpr
 	}
 	checkItems := strings.Split(checkExpr, ".")
 	for i, checkItem := range checkItems {
-		if strings.Contains(checkItem, "-") && !strings.Contains(checkItem, "\"") {
+		checkItem = strings.Trim(checkItem, "\"")
+		lowerItem := strings.ToLower(checkItem)
+		if strings.HasPrefix(lowerItem, "content-") || lowerItem == "user-agent" {
 			checkItems[i] = fmt.Sprintf("\"%s\"", checkItem)
 		}
 	}
--- a/hrp/testcase_test.go
+++ b/hrp/testcase_test.go
@@ -210,21 +210,20 @@ func TestConvertCheckExpr(t *testing.T) {
 	}{
 		// normal check expression
 		{"a.b.c", "a.b.c"},
-		{"headers.\"Content-Type\"", "headers.\"Content-Type\""},
+		{"a.\"b-c\".d", "a.\"b-c\".d"},
+		{"a.b-c.d", "a.b-c.d"},
+		{"body.args.a[-1]", "body.args.a[-1]"},
 		// check expression using regex
 		{"covering (.*) testing,", "covering (.*) testing,"},
 		{" (.*) a-b-c", " (.*) a-b-c"},
 		// abnormal check expression
-		{"-", "\"-\""},
-		{"b-c", "\"b-c\""},
-		{"a.b-c.d", "a.\"b-c\".d"},
-		{"a-b.c-d", "\"a-b\".\"c-d\""},
-		{"\"a-b\".c-d", "\"a-b\".\"c-d\""},
 		{"headers.Content-Type", "headers.\"Content-Type\""},
-		{"body.I-am-a-Key.name", "body.\"I-am-a-Key\".name"},
+		{"headers.\"Content-Type", "headers.\"Content-Type\""},
+		{"headers.Content-Type\"", "headers.\"Content-Type\""},
+		{"headers.User-Agent", "headers.\"User-Agent\""},
 	}
 	for _, expr := range exprs {
-		if !assert.Equal(t, convertCheckExpr(expr.before), expr.after) {
+		if !assert.Equal(t, expr.after, convertJmespathExpr(expr.before)) {
 			t.Fatal()
 		}
 	}
--- a/hrp/tests/extract_test.go
+++ b/hrp/tests/extract_test.go
@@ -18,6 +18,7 @@ func TestCaseExtractStepSingle(t *testing.T) {
 					"var1":               "bar1",
 					"agent":              "HttpRunnerPlus",
 					"expectedStatusCode": 200,
+					"jmespathFoo1":       "body.args.foo1",
 				}).
 				GET("/get").
 				WithParams(map[string]interface{}{"foo1": "$var1", "foo2": "bar2"}).
@@ -25,7 +26,7 @@ func TestCaseExtractStepSingle(t *testing.T) {
 				Extract().
 				WithJmesPath("status_code", "statusCode").
 				WithJmesPath("headers.\"Content-Type\"", "contentType").
-				WithJmesPath("body.args.foo1", "varFoo1").
+				WithJmesPath("$jmespathFoo1", "varFoo1").
 				Validate().
 				AssertEqual("$statusCode", "$expectedStatusCode", "check status code").                      // assert with extracted variable from current step
 				AssertEqual("$contentType", "application/json; charset=utf-8", "check header Content-Type"). // assert with extracted variable from current step
--- a/httprunner/compat.py
+++ b/httprunner/compat.py
@@ -14,7 +14,7 @@ from httprunner.utils import sort_dict_by_custom_order


 def convert_variables(
-        raw_variables: Union[Dict, Text], test_path: Text
+    raw_variables: Union[Dict, Text], test_path: Text
 ) -> Dict[Text, Any]:
    if isinstance(raw_variables, Dict):
        return raw_variables
@@ -54,24 +54,13 @@ def _convert_jmespath(raw: Text) -> Text:
    elif raw.startswith("json"):
        raw = f"body{raw[len('json'):]}"

-    raw_list = []
-    for item in raw.split("."):
+    raw_list = raw.split(".")
+    for i, item in enumerate(raw_list):
+        item = item.strip('"')
        if item.lower().startswith("content-") or item.lower() in ["user-agent"]:
            # add quotes for some field in white list
            # e.g. headers.Content-Type => headers."Content-Type"
-            item = item.strip('"')
-            raw_list.append(f'"{item}"')
-        elif item.isdigit():
-            # convert lst.0.name to lst[0].name
-            if len(raw_list) == 0:
-                logger.error(f"Invalid jmespath: {raw}")
-                sys.exit(1)
-
-            last_item = raw_list.pop()
-            item = f"{last_item}[{item}]"
-            raw_list.append(item)
-        else:
-            raw_list.append(item)
+            raw_list[i] = f'"{item}"'

    return ".".join(raw_list)

--- a/httprunner/compat_test.py
+++ b/httprunner/compat_test.py
@@ -30,47 +30,33 @@ class TestCompat(unittest.TestCase):
        request_with_json_body = {
            "method": "POST",
            "url": "https://postman-echo.com/post",
-            "headers": {
-                "Content-Type": "application/json"
-            },
-            "body": {
-                "k1": "v1",
-                "k2": "v2"
-            }
+            "headers": {"Content-Type": "application/json"},
+            "body": {"k1": "v1", "k2": "v2"},
        }
        self.assertEqual(
            compat._convert_request(request_with_json_body),
            {
                "method": "POST",
                "url": "https://postman-echo.com/post",
-                "headers": {
-                    "Content-Type": "application/json"
-                },
-                "json": {
-                    "k1": "v1",
-                    "k2": "v2"
-                }
-            }
+                "headers": {"Content-Type": "application/json"},
+                "json": {"k1": "v1", "k2": "v2"},
+            },
        )

        request_with_text_body = {
            "method": "POST",
            "url": "https://postman-echo.com/post",
-            "headers": {
-                "Content-Type": "text/plain"
-            },
-            "body": "have a nice day"
+            "headers": {"Content-Type": "text/plain"},
+            "body": "have a nice day",
        }
        self.assertEqual(
            compat._convert_request(request_with_text_body),
            {
                "method": "POST",
                "url": "https://postman-echo.com/post",
-                "headers": {
-                    "Content-Type": "text/plain"
-                },
-                "data": "have a nice day"
-            }
+                "headers": {"Content-Type": "text/plain"},
+                "data": "have a nice day",
+            },
        )

    def test_convert_jmespath(self):
@@ -85,10 +71,6 @@ class TestCompat(unittest.TestCase):
        self.assertEqual(
            compat._convert_jmespath('headers."Content-Type"'), 'headers."Content-Type"'
        )
-        self.assertEqual(
-            compat._convert_jmespath("body.data.buildings.0.building_id"),
-            "body.data.buildings[0].building_id",
-        )
        self.assertEqual(
            compat._convert_jmespath("body.users[-1]"),
            "body.users[-1]",
@@ -97,8 +79,6 @@ class TestCompat(unittest.TestCase):
            compat._convert_jmespath("body.result.WorkNode_-1"),
            "body.result.WorkNode_-1",
        )
-        with self.assertRaises(SystemExit):
-            compat._convert_jmespath("2.buildings.0.building_id")

    def test_convert_extractors(self):
        self.assertEqual(
@@ -108,11 +88,11 @@ class TestCompat(unittest.TestCase):
            {"varA": "body.varA", "varB": "body.varB"},
        )
        self.assertEqual(
-            compat._convert_extractors([{"varA": "content.0.varA"}]),
+            compat._convert_extractors([{"varA": "content[0].varA"}]),
            {"varA": "body[0].varA"},
        )
        self.assertEqual(
-            compat._convert_extractors({"varA": "content.0.varA"}),
+            compat._convert_extractors({"varA": "content[0].varA"}),
            {"varA": "body[0].varA"},
        )

@@ -128,7 +108,7 @@ class TestCompat(unittest.TestCase):
            [{"eq": ["body.abc", 201]}],
        )
        self.assertEqual(
-            compat._convert_validators([{"eq": ["content.0.name", 201]}]),
+            compat._convert_validators([{"eq": ["content[0].name", 201]}]),
            [{"eq": ["body[0].name", 201]}],
        )

@@ -142,7 +122,7 @@ class TestCompat(unittest.TestCase):
                "headers": {"User-Agent": "HttpRunner/3.0"},
            },
            "extract": [{"varA": "content.varA"}, {"user_agent": "headers.User-Agent"}],
-            "validate": [{"eq": ["content.varB", 200]}, {"lt": ["json.0.varC", 0]}],
+            "validate": [{"eq": ["content.varB", 200]}, {"lt": ["json[0].varC", 0]}],
        }
        self.assertEqual(
            compat.ensure_testcase_v4_api(api_content),
@@ -191,7 +171,7 @@ class TestCompat(unittest.TestCase):
                    ],
                    "validate": [
                        {"eq": ["content.varB", 200]},
-                        {"lt": ["json.0.varC", 0]},
+                        {"lt": ["json[0].varC", 0]},
                    ],
                }
            ],