refactor: implement parse_string to eval string with both variables and functions

httprunner/v3/parser.py

@@ -107,56 +107,6 @@ def regex_findall_functions(content: Text) -> List[Text]:
         return []
 
 
-def parse_args_str(arg_str: Text) -> Tuple[List, Dict]:
-    """ parse function args and kwargs from function.
-
-    Args:
-        arg_str (str): function str contains args and kwargs
-
-    Returns:
-        dict: function meta dict
-
-            {
-                "func_name": "xxx",
-                "args": [],
-                "kwargs": {}
-            }
-
-    Examples:
-        >>> parse_args_str("")
-        {'args': [], 'kwargs': {}}
-
-        >>> parse_args_str("5")
-        {'args': [5], 'kwargs': {}}
-
-        >>> parse_args_str("1, 2")
-        {'args': [1, 2], 'kwargs': {}}
-
-        >>> parse_args_str("a=1, b=2")
-        {'args': [], 'kwargs': {'a': 1, 'b': 2}}
-
-        >>> parse_args_str("1, 2, a=3, b=4")
-        {'args': [1, 2], 'kwargs': {'a':3, 'b':4}}
-
-    """
-    args = []
-    kwargs = {}
-    arg_str = arg_str.strip()
-    if arg_str == "":
-        return args, kwargs
-
-    arg_list = arg_str.split(',')
-    for arg in arg_list:
-        arg = arg.strip()
-        if '=' in arg:
-            key, value = arg.split('=')
-            kwargs[key.strip()] = parse_string_value(value.strip())
-        else:
-            args.append(parse_string_value(arg))
-
-    return args, kwargs
-
-
 def extract_variables(content: Any) -> Set:
     """ extract all variables in content recursively.
     """
@@ -178,97 +128,156 @@ def extract_variables(content: Any) -> Set:
     return set()
 
 
-def parse_string_functions(
+def parse_function_params(params):
-        content: Text,
+    """ parse function params to args and kwargs.
-        variables_mapping: Dict[Text, Any],
-        functions_mapping: Dict[Text, Callable]) -> Text:
-    """ parse string content with functions mapping.
 
     Args:
-        content (str): string content to be parsed.
+        params (str): function param in string
-        variables_mapping (dict): variables mapping.
+
-        functions_mapping (dict): functions mapping.
+    Returns:
+        dict: function meta dict
+
+            {
+                "args": [],
+                "kwargs": {}
+            }
+
+    Examples:
+        >>> parse_function_params("")
+        {'args': [], 'kwargs': {}}
+
+        >>> parse_function_params("5")
+        {'args': [5], 'kwargs': {}}
+
+        >>> parse_function_params("1, 2")
+        {'args': [1, 2], 'kwargs': {}}
+
+        >>> parse_function_params("a=1, b=2")
+        {'args': [], 'kwargs': {'a': 1, 'b': 2}}
+
+        >>> parse_function_params("1, 2, a=3, b=4")
+        {'args': [1, 2], 'kwargs': {'a':3, 'b':4}}
+
+    """
+    function_meta = {
+        "args": [],
+        "kwargs": {}
+    }
+
+    params_str = params.strip()
+    if params_str == "":
+        return function_meta
+
+    args_list = params_str.split(',')
+    for arg in args_list:
+        arg = arg.strip()
+        if '=' in arg:
+            key, value = arg.split('=')
+            function_meta["kwargs"][key.strip()] = parse_string_value(value.strip())
+        else:
+            function_meta["args"].append(parse_string_value(arg))
+
+    return function_meta
+
+
+def parse_string(
+        raw_string: Text,
+        variables_mapping: Dict[Text, Any],
+        functions_mapping: Dict[Text, Callable]) -> Text:
+    """ parse string content with variables and functions mapping.
+
+    Args:
+        raw_string: raw string content to be parsed.
+        variables_mapping: variables mapping.
+        functions_mapping: functions mapping.
 
     Returns:
         str: parsed string content.
 
     Examples:
-        >>> content = "abc${add_one(3)}def"
+        >>> raw_string = "abc${add_one($num)}def"
+        >>> variables_mapping = {"num": 3}
         >>> functions_mapping = {"add_one": lambda x: x + 1}
-        >>> parse_string_functions(content, {}, functions_mapping)
+        >>> parse_string(raw_string, variables_mapping, functions_mapping)
             "abc4def"
 
     """
-    functions_list = regex_findall_functions(content)
+    try:
-    for func_meta_tuple in functions_list:
+        match_start_position = raw_string.index("$", 0)
-        func_name, args_str = func_meta_tuple
+        parsed_string = raw_string[0:match_start_position]
-        args, kwargs = parse_args_str(args_str)
+    except ValueError:
+        parsed_string = raw_string
+        return parsed_string
 
-        args = parse_content(args, variables_mapping, functions_mapping)
+    while match_start_position < len(raw_string):
-        kwargs = parse_content(kwargs, variables_mapping, functions_mapping)
 
+        # Notice: notation priority
+        # $$ > ${func($a, $b)} > $var
+
+        # search $$
+        dollar_match = dolloar_regex_compile.match(raw_string, match_start_position)
+        if dollar_match:
+            match_start_position = dollar_match.end()
+            parsed_string += "$"
+            continue
+
+        # search function like ${func($a, $b)}
+        func_match = function_regex_compile.match(raw_string, match_start_position)
+        if func_match:
+            func_name = func_match.group(1)
+            try:
+                func = functions_mapping[func_name]
+            except KeyError:
+                raise FunctionNotFound(f"{func_name} not found in {functions_mapping}")
+
+            func_params_str = func_match.group(2)
+            function_meta = parse_function_params(func_params_str)
+            args = function_meta["args"]
+            kwargs = function_meta["kwargs"]
+            func_eval_value = func(*args, **kwargs)
+
+            func_raw_str = "${" + func_name + f"({func_params_str})" + "}"
+            if func_raw_str == raw_string:
+                # raw_string is a function, e.g. "${add_one(3)}", return its eval value directly
+                return func_eval_value
+
+            # raw_string contains one or many functions, e.g. "abc${add_one(3)}def"
+            parsed_string += str(func_eval_value)
+            match_start_position = func_match.end()
+            continue
+
+        # search variable like ${var} or $var
+        var_match = variable_regex_compile.match(raw_string, match_start_position)
+        if var_match:
+            var_name = var_match.group(1) or var_match.group(2)
+            # check if any variable undefined in variables_mapping
+            try:
+                var_value = variables_mapping[var_name]
+            except KeyError:
+                raise VariableNotFound(f"{var_name} not found in {variables_mapping}")
+
+            if f"${var_name}" == raw_string or "${" + var_name + "}" == raw_string:
+                # raw_string is a variable, $var or ${var}, return its value directly
+                return var_value
+
+            # raw_string contains one or many variables, e.g. "abc${var}def"
+            parsed_string += str(var_value)
+            match_start_position = var_match.end()
+            continue
+
+        curr_position = match_start_position
         try:
-            func = functions_mapping[func_name]
+            # find next $ location
-        except KeyError:
+            match_start_position = raw_string.index("$", curr_position + 1)
-            raise FunctionNotFound(f"{func_name} not found in {functions_mapping}")
+            remain_string = raw_string[curr_position:match_start_position]
+        except ValueError:
+            remain_string = raw_string[curr_position:]
+            # break while loop
+            match_start_position = len(raw_string)
 
-        eval_value = func(*args, **kwargs)
+        parsed_string += remain_string
 
-        func_content = "${" + func_name + f"({args_str})" + "}"
+    return parsed_string
-        if func_content == content:
-            # content is a function, e.g. "${add_one(3)}"
-            content = eval_value
-        else:
-            # content contains one or many functions, e.g. "abc${add_one(3)}def"
-            content = content.replace(
-                func_content,
-                str(eval_value), 1
-            )
-
-    return content
-
-
-def parse_string_variables(
-        content: Text,
-        variables_mapping: Dict[Text, Any]) -> Text:
-    """ parse string content with variables mapping.
-
-    Args:
-        content (str): string content to be parsed.
-        variables_mapping (dict): variables mapping.
-
-    Returns:
-        str: parsed string content.
-
-    Examples:
-        >>> content = "/api/users/$uid"
-        >>> variables_mapping = {"uid": 1000}
-        >>> parse_string_variables(content, variables_mapping)
-            "/api/users/1000"
-
-    """
-    variables_list = extract_variables(content)
-    for variable_name in variables_list:
-        try:
-            variable_value = variables_mapping[variable_name]
-        except KeyError:
-            raise VariableNotFound(f"{variable_name} not found in {variables_mapping}")
-
-        # TODO: replace variable label from $var to {{var}}
-        if f"${variable_name}" == content:
-            # content is a variable
-            content = variable_value
-        else:
-            # content contains one or several variables
-            if not isinstance(variable_value, str):
-                variable_value = str(variable_value)
-
-            content = content.replace(
-                f"${variable_name}",
-                variable_value, 1
-            )
-
-    return content
 
 
 def parse_content(
@@ -278,24 +287,20 @@ def parse_content(
     """ parse content with evaluated variables mapping.
         Notice: variables_mapping should not contain any variable or function.
     """
-    # TODO: refactor type check
+    if isinstance(content, str):
-    if content is None or isinstance(content, (int, float, bool)):
+        # content in string format may contains variables and functions
-        return content
-
-    elif isinstance(content, str):
-        # content is in string format here
         variables_mapping = variables_mapping or {}
         functions_mapping = functions_mapping or {}
         content = content.strip()
 
         # replace functions with evaluated value
         # Notice: parse_string_functions must be called before parse_string_variables
-        content = parse_string_functions(content, variables_mapping, functions_mapping)
+        # content = parse_string_functions(content, variables_mapping, functions_mapping)
 
         # replace variables with binding value
-        content = parse_string_variables(content, variables_mapping)
+        # content = parse_string_variables(content, variables_mapping)
 
-        return content
+        return parse_string(content, variables_mapping, functions_mapping)
 
     elif isinstance(content, (list, set, tuple)):
         return [
@@ -312,7 +317,9 @@ def parse_content(
 
         return parsed_content
 
-    return content
+    else:
+        # other types, e.g. None, int, float, bool
+        return content
 
 
 def parse_variables_mapping(

httprunner/v3/parser_test.py

@@ -83,46 +83,46 @@ class TestParserBasic(unittest.TestCase):
             {"TOKEN", "data", "random"}
         )
 
-    def test_parse_function(self):
+    def test_parse_function_params(self):
         self.assertEqual(
-            parser.parse_args_str(""),
+            parser.parse_function_params(""),
-            ([], {})
+            {'args': [], 'kwargs': {}}
         )
         self.assertEqual(
-            parser.parse_args_str("5"),
+            parser.parse_function_params("5"),
-            ([5], {})
+            {'args': [5], 'kwargs': {}}
         )
         self.assertEqual(
-            parser.parse_args_str("1, 2"),
+            parser.parse_function_params("1, 2"),
-            ([1, 2], {})
+            {'args': [1, 2], 'kwargs': {}}
         )
         self.assertEqual(
-            parser.parse_args_str("a=1, b=2"),
+            parser.parse_function_params("a=1, b=2"),
-            ([], {'a': 1, 'b': 2})
+            {'args': [], 'kwargs': {'a': 1, 'b': 2}}
         )
         self.assertEqual(
-            parser.parse_args_str("a= 1, b =2"),
+            parser.parse_function_params("a= 1, b =2"),
-            ([], {'a': 1, 'b': 2})
+            {'args': [], 'kwargs': {'a': 1, 'b': 2}}
         )
         self.assertEqual(
-            parser.parse_args_str("1, 2, a=3, b=4"),
+            parser.parse_function_params("1, 2, a=3, b=4"),
-            ([1, 2], {'a': 3, 'b': 4})
+            {'args': [1, 2], 'kwargs': {'a': 3, 'b': 4}}
         )
         self.assertEqual(
-            parser.parse_args_str("$request, 123"),
+            parser.parse_function_params("$request, 123"),
-            (["$request", 123], {})
+            {'args': ["$request", 123], 'kwargs': {}}
         )
         self.assertEqual(
-            parser.parse_args_str("  "),
+            parser.parse_function_params("  "),
-            ([], {})
+            {'args': [], 'kwargs': {}}
         )
         self.assertEqual(
-            parser.parse_args_str("hello world, a=3, b=4"),
+            parser.parse_function_params("hello world, a=3, b=4"),
-            (["hello world"], {'a': 3, 'b': 4})
+            {'args': ["hello world"], 'kwargs': {'a': 3, 'b': 4}}
         )
         self.assertEqual(
-            parser.parse_args_str("$request, 12 3"),
+            parser.parse_function_params("$request, 12 3"),
-            (["$request", '12 3'], {})
+            {'args': ["$request", '12 3'], 'kwargs': {}}
         )
 
     def test_extract_functions(self):
@@ -192,7 +192,7 @@ class TestParserBasic(unittest.TestCase):
         self.assertEqual("", result["request"]["data"]["empty_str"])
         self.assertEqual("abc4def", result["request"]["data"]["value"])
 
-    def test_parse_data_variables(self):
+    def test_parse_content_with_variables(self):
         variables_mapping = {
             "var_1": "abc",
             "var_2": "def",
@@ -205,6 +205,10 @@ class TestParserBasic(unittest.TestCase):
             parser.parse_content("$var_1", variables_mapping),
             "abc"
         )
+        self.assertEqual(
+            parser.parse_content("${var_1}", variables_mapping),
+            "abc"
+        )
         self.assertEqual(
             parser.parse_content("var_1", variables_mapping),
             "var_1"
@@ -214,12 +218,12 @@ class TestParserBasic(unittest.TestCase):
             "abc#XYZ"
         )
         self.assertEqual(
-            parser.parse_content("/$var_1/$var_2/var3", variables_mapping),
+            parser.parse_content("${var_1}#XYZ", variables_mapping),
-            "/abc/def/var3"
+            "abc#XYZ"
         )
         self.assertEqual(
-            parser.parse_string_variables("${func($var_1, $var_2, xyz)}", variables_mapping),
+            parser.parse_content("/$var_1/$var_2/var3", variables_mapping),
-            "${func(abc, def, xyz)}"
+            "/abc/def/var3"
         )
         self.assertEqual(
             parser.parse_content("$var_3", variables_mapping),
@@ -258,6 +262,37 @@ class TestParserBasic(unittest.TestCase):
             {"abc": "def"}
         )
 
+    def test_parse_data_multiple_identical_variables(self):
+        variables_mapping = {
+            "var_1": "abc",
+            "var_2": "def",
+        }
+        self.assertEqual(
+            parser.parse_content("/$var_1/$var_2/$var_1", variables_mapping),
+            "/abc/def/abc"
+        )
+
+        variables_mapping = {
+            "userid": 100,
+            "data": 1498
+        }
+        content = "/users/$userid/training/$data?userId=$userid&data=$data"
+        self.assertEqual(
+            parser.parse_content(content, variables_mapping),
+            "/users/100/training/1498?userId=100&data=1498"
+        )
+
+        variables_mapping = {
+            "user": 100,
+            "userid": 1000,
+            "data": 1498
+        }
+        content = "/users/$user/$userid/$data?userId=$userid&data=$data"
+        self.assertEqual(
+            parser.parse_content(content, variables_mapping),
+            "/users/100/1000/1498?userId=1000&data=1498"
+        )
+
     def test_parse_data_functions(self):
         import random, string
         functions_mapping = {