refactor parse_data

httprunner/loader.py

@@ -501,7 +501,7 @@ def _get_block_by_name(ref_call, ref_type):
         args_mapping[item] = call_args[index]
 
     if args_mapping:
-        block = parser.parse_data(block, args_mapping)
+        block = parser.substitute_variables(block, args_mapping)
 
     return block
 

httprunner/parser.py

@@ -5,7 +5,7 @@ import os
 import re
 
 from httprunner import exceptions
-from httprunner.compat import builtin_str, numeric_types, str
+from httprunner.compat import basestring, builtin_str, numeric_types, str
 
 variable_regexp = r"\$([\w_]+)"
 function_regexp = r"\$\{([\w_]+\([\$\w\.\-_ =,]*\))\}"
@@ -200,12 +200,205 @@ def parse_validator(validator):
     }
 
 
-def parse_data(content, variables_mapping=None):
+def substitute_variables(content, variables_mapping):
+    """ substitute variables in content with variables_mapping
+
+    Args:
+        content (str/dict/list/numeric/bool/type): content to be substituted.
+        variables_mapping (dict): variables mapping.
+
+    Returns:
+        substituted content.
+
+    Examples:
+        >>> content = {
+                'request': {
+                    'url': '/api/users/$uid',
+                    'headers': {'token': '$token'}
+                }
+            }
+        >>> variables_mapping = {"$uid": 1000}
+        >>> substitute_variables(content, variables_mapping)
+            {
+                'request': {
+                    'url': '/api/users/1000',
+                    'headers': {'token': '$token'}
+                }
+            }
+
+    """
+    if isinstance(content, (list, set, tuple)):
+        return [
+            substitute_variables(item, variables_mapping)
+            for item in content
+        ]
+
+    if isinstance(content, dict):
+        substituted_data = {}
+        for key, value in content.items():
+            eval_key = substitute_variables(key, variables_mapping)
+            eval_value = substitute_variables(value, variables_mapping)
+            substituted_data[eval_key] = eval_value
+
+        return substituted_data
+
+    if isinstance(content, basestring):
+        # content is in string format here
+        for var, value in variables_mapping.items():
+            if content == var:
+                # content is a variable
+                content = value
+            else:
+                if not isinstance(value, str):
+                    value = builtin_str(value)
+                content = content.replace(var, value)
+
+    return content
+
+
+###############################################################################
+##  parse content with variables and functions mapping
+###############################################################################
+
+def get_mapping_variable(variable_name, variables_mapping):
+    """ get variable from variables_mapping.
+
+    Args:
+        variable_name (str): variable name
+        variables_mapping (dict): variables mapping
+
+    Returns:
+        mapping variable value.
+
+    Raises:
+        exceptions.VariableNotFound: variable is not found.
+
+    """
+    try:
+        return variables_mapping[variable_name]
+    except KeyError:
+        raise exceptions.VariableNotFound("{} is not found.".format(variable_name))
+
+
+def get_mapping_function(function_name, functions_mapping):
+    """ get function from functions_mapping,
+        if not found, then try to check if builtin function.
+
+    Args:
+        variable_name (str): variable name
+        variables_mapping (dict): variables mapping
+
+    Returns:
+        mapping function object.
+
+    Raises:
+        exceptions.FunctionNotFound: function is neither defined in debugtalk.py nor builtin.
+
+    """
+    if function_name in functions_mapping:
+        return functions_mapping[function_name]
+
+    try:
+        # check if builtin functions
+        item_func = eval(function_name)
+        if callable(item_func):
+            # is builtin function
+            return item_func
+    except (NameError, TypeError):
+        # is not builtin function
+        raise exceptions.FunctionNotFound("{} is not found.".format(function_name))
+
+
+def parse_string_functions(content, variables_mapping, functions_mapping):
+    """ parse string content with functions mapping.
+
+    Args:
+        content (str): string content to be parsed.
+        variables_mapping (dict): variables mapping.
+        functions_mapping (dict): functions mapping.
+
+    Returns:
+        str: parsed string content.
+
+    Examples:
+        >>> content = "abc${add_one(3)}def"
+        >>> functions_mapping = {"add_one": lambda x: x + 1}
+        >>> parse_string_functions(content, functions_mapping)
+            "abc4def"
+
+    """
+    functions_list = extract_functions(content)
+    for func_content in functions_list:
+        function_meta = parse_function(func_content)
+        func_name = function_meta["func_name"]
+
+        args = function_meta.get("args", [])
+        kwargs = function_meta.get("kwargs", {})
+        args = parse_data(args, variables_mapping, functions_mapping)
+        kwargs = parse_data(kwargs, variables_mapping, functions_mapping)
+
+        func = get_mapping_function(func_name, functions_mapping)
+        eval_value = func(*args, **kwargs)
+
+        func_content = "${" + func_content + "}"
+        if func_content == content:
+            # content is a function, e.g. "${add_one(3)}"
+            content = eval_value
+        else:
+            # content contains one or many functions, e.g. "abc${add_one(3)}def"
+            content = content.replace(
+                func_content,
+                str(eval_value), 1
+            )
+
+    return content
+
+
+def parse_string_variables(content, variables_mapping):
+    """ parse string content with variables mapping.
+
+    Args:
+        content (str): string content to be parsed.
+        variables_mapping (dict): variables mapping.
+
+    Returns:
+        str: parsed string content.
+
+    Examples:
+        >>> content = "/api/users/$uid"
+        >>> variables_mapping = {"$uid": 1000}
+        >>> parse_string_variables(content, variables_mapping)
+            "/api/users/1000"
+
+    """
+    variables_list = extract_variables(content)
+    for variable_name in variables_list:
+        variable_value = get_mapping_variable(variable_name, variables_mapping)
+
+        # TODO: replace variable label from $var to {{var}}
+        if "${}".format(variable_name) == content:
+            # content is a variable
+            content = variable_value
+        else:
+            # content contains one or several variables
+            if not isinstance(variable_value, str):
+                variable_value = builtin_str(variable_value)
+
+            content = content.replace(
+                "${}".format(variable_name),
+                variable_value, 1
+            )
+
+    return content
+
+
+def parse_data(content, variables_mapping=None, functions_mapping=None):
     """ parse content with variables mapping
 
     Args:
         content (str/dict/list/numeric/bool/type): content to be parsed
-        variables_mapping (dict): variables mapping
+        variables_mapping (dict): variables mapping.
+        functions_mapping (dict): functions mapping.
 
     Returns:
         parsed content.
@@ -217,12 +410,12 @@ def parse_data(content, variables_mapping=None):
                     'headers': {'token': '$token'}
                 }
             }
-        >>> variables_mapping = {"$uid": 1000}
+        >>> variables_mapping = {"uid": 1000, "token": "abcdef"}
         >>> parse_data(content, variables_mapping)
             {
                 'request': {
                     'url': '/api/users/1000',
-                    'headers': {'token': '$token'}
+                    'headers': {'token': 'abcdef'}
                 }
             }
 
@@ -234,28 +427,30 @@ def parse_data(content, variables_mapping=None):
 
     if isinstance(content, (list, set, tuple)):
         return [
-            parse_data(item, variables_mapping)
+            parse_data(item, variables_mapping, functions_mapping)
             for item in content
         ]
 
     if isinstance(content, dict):
-        parsed_data = {}
+        parsed_content = {}
         for key, value in content.items():
-            eval_key = parse_data(key, variables_mapping)
-            eval_value = parse_data(value, variables_mapping)
-            parsed_data[eval_key] = eval_value
+            parsed_key = parse_data(key, variables_mapping, functions_mapping)
+            parsed_value = parse_data(value, variables_mapping, functions_mapping)
+            parsed_content[parsed_key] = parsed_value
 
-        return parsed_data
+        return parsed_content
 
-    # content is in string format here
-    variables_mapping = variables_mapping or {}
-    for var, value in variables_mapping.items():
-        if content == var:
-            # content is a variable
-            content = value
-        else:
-            if not isinstance(value, str):
-                value = builtin_str(value)
-            content = content.replace(var, value)
+    if isinstance(content, basestring):
+        # content is in string format here
+        variables_mapping = variables_mapping or {}
+        functions_mapping = functions_mapping or {}
+        content = content.strip()
+
+        # replace functions with evaluated value
+        # Notice: _eval_content_functions must be called before _eval_content_variables
+        content = parse_string_functions(content, variables_mapping, functions_mapping)
+
+        # replace variables with binding value
+        content = parse_string_variables(content, variables_mapping)
 
     return content

tests/test_context.py

@@ -409,40 +409,6 @@ class TestTestcaseParser(unittest.TestCase):
             3
         )
 
-    def test_extract_functions(self):
-        self.assertEqual(
-            parser.extract_functions("${func()}"),
-            ["func()"]
-        )
-        self.assertEqual(
-            parser.extract_functions("${func(5)}"),
-            ["func(5)"]
-        )
-        self.assertEqual(
-            parser.extract_functions("${func(a=1, b=2)}"),
-            ["func(a=1, b=2)"]
-        )
-        self.assertEqual(
-            parser.extract_functions("${func(1, $b, c=$x, d=4)}"),
-            ["func(1, $b, c=$x, d=4)"]
-        )
-        self.assertEqual(
-            parser.extract_functions("/api/1000?_t=${get_timestamp()}"),
-            ["get_timestamp()"]
-        )
-        self.assertEqual(
-            parser.extract_functions("/api/${add(1, 2)}"),
-            ["add(1, 2)"]
-        )
-        self.assertEqual(
-            parser.extract_functions("/api/${add(1, 2)}?_t=${get_timestamp()}"),
-            ["add(1, 2)", "get_timestamp()"]
-        )
-        self.assertEqual(
-            parser.extract_functions("abc${func(1, 2, a=3, b=4)}def"),
-            ["func(1, 2, a=3, b=4)"]
-        )
-
     def test_eval_content_functions(self):
         functions = {
             "add_two_nums": lambda a, b=1: a + b

tests/test_parser.py

@@ -115,6 +115,40 @@ class TestParser(unittest.TestCase):
             {"check": "status_code", "comparator": "eq", "expect": 201}
         )
 
+    def test_extract_functions(self):
+        self.assertEqual(
+            parser.extract_functions("${func()}"),
+            ["func()"]
+        )
+        self.assertEqual(
+            parser.extract_functions("${func(5)}"),
+            ["func(5)"]
+        )
+        self.assertEqual(
+            parser.extract_functions("${func(a=1, b=2)}"),
+            ["func(a=1, b=2)"]
+        )
+        self.assertEqual(
+            parser.extract_functions("${func(1, $b, c=$x, d=4)}"),
+            ["func(1, $b, c=$x, d=4)"]
+        )
+        self.assertEqual(
+            parser.extract_functions("/api/1000?_t=${get_timestamp()}"),
+            ["get_timestamp()"]
+        )
+        self.assertEqual(
+            parser.extract_functions("/api/${add(1, 2)}"),
+            ["add(1, 2)"]
+        )
+        self.assertEqual(
+            parser.extract_functions("/api/${add(1, 2)}?_t=${get_timestamp()}"),
+            ["add(1, 2)", "get_timestamp()"]
+        )
+        self.assertEqual(
+            parser.extract_functions("abc${func(1, 2, a=3, b=4)}def"),
+            ["func(1, 2, a=3, b=4)"]
+        )
+
     def test_parse_data(self):
         content = {
             'request': {
@@ -125,19 +159,200 @@ class TestParser(unittest.TestCase):
                     "null": None,
                     "true": True,
                     "false": False,
-                    "empty_str": ""
+                    "empty_str": "",
+                    "value": "abc${add_one(3)}def"
                 }
             }
         }
-        mapping = {
-            "$uid": 1000,
-            "$method": "POST"
+        variables_mapping = {
+            "uid": 1000,
+            "method": "POST",
+            "token": "abc123"
         }
-        result = parser.parse_data(content, mapping)
+        functions_mapping = {
+            "add_one": lambda x: x + 1
+        }
+        result = parser.parse_data(content, variables_mapping, functions_mapping)
         self.assertEqual("/api/users/1000", result["request"]["url"])
-        self.assertEqual("$token", result["request"]["headers"]["token"])
+        self.assertEqual("abc123", result["request"]["headers"]["token"])
         self.assertEqual("POST", result["request"]["method"])
         self.assertIsNone(result["request"]["data"]["null"])
         self.assertTrue(result["request"]["data"]["true"])
         self.assertFalse(result["request"]["data"]["false"])
         self.assertEqual("", result["request"]["data"]["empty_str"])
+        self.assertEqual("abc4def", result["request"]["data"]["value"])
+
+    def test_parse_data_variables(self):
+        variables_mapping = {
+            "var_1": "abc",
+            "var_2": "def",
+            "var_3": 123,
+            "var_4": {"a": 1},
+            "var_5": True,
+            "var_6": None
+        }
+        self.assertEqual(
+            parser.parse_data("$var_1", variables_mapping),
+            "abc"
+        )
+        self.assertEqual(
+            parser.parse_data("var_1", variables_mapping),
+            "var_1"
+        )
+        self.assertEqual(
+            parser.parse_data("$var_1#XYZ", variables_mapping),
+            "abc#XYZ"
+        )
+        self.assertEqual(
+            parser.parse_data("/$var_1/$var_2/var3", variables_mapping),
+            "/abc/def/var3"
+        )
+        self.assertEqual(
+            parser.parse_data("/$var_1/$var_2/$var_1", variables_mapping),
+            "/abc/def/abc"
+        )
+        self.assertEqual(
+            parser.parse_string_variables("${func($var_1, $var_2, xyz)}", variables_mapping),
+            "${func(abc, def, xyz)}"
+        )
+        self.assertEqual(
+            parser.parse_data("$var_3", variables_mapping),
+            123
+        )
+        self.assertEqual(
+            parser.parse_data("$var_4", variables_mapping),
+            {"a": 1}
+        )
+        self.assertEqual(
+            parser.parse_data("$var_5", variables_mapping),
+            True
+        )
+        self.assertEqual(
+            parser.parse_data("abc$var_5", variables_mapping),
+            "abcTrue"
+        )
+        self.assertEqual(
+            parser.parse_data("abc$var_4", variables_mapping),
+            "abc{'a': 1}"
+        )
+        self.assertEqual(
+            parser.parse_data("$var_6", variables_mapping),
+            None
+        )
+
+        with self.assertRaises(exceptions.VariableNotFound):
+            parser.parse_data("/api/$SECRET_KEY", variables_mapping)
+
+        self.assertEqual(
+            parser.parse_data(["$var_1", "$var_2"], variables_mapping),
+            ["abc", "def"]
+        )
+        self.assertEqual(
+            parser.parse_data({"$var_1": "$var_2"}, variables_mapping),
+            {"abc": "def"}
+        )
+
+    def test_parse_data_multiple_identical_variables(self):
+        variables_mapping = {
+            "userid": 100,
+            "data": 1498
+        }
+        content = "/users/$userid/training/$data?userId=$userid&data=$data"
+        self.assertEqual(
+            parser.parse_data(content, variables_mapping),
+            "/users/100/training/1498?userId=100&data=1498"
+        )
+
+        variables_mapping = {
+            "user": 100,
+            "userid": 1000,
+            "data": 1498
+        }
+        content = "/users/$user/$userid/$data?userId=$userid&data=$data"
+        self.assertEqual(
+            parser.parse_data(content, variables_mapping),
+            "/users/100/1000/1498?userId=1000&data=1498"
+        )
+
+    def test_parse_data_functions(self):
+        import random, string
+        functions_mapping = {
+            "gen_random_string": lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) \
+                for _ in range(str_len))
+        }
+        result = parser.parse_data("${gen_random_string(5)}", functions_mapping=functions_mapping)
+        self.assertEqual(len(result), 5)
+
+        add_two_nums = lambda a, b=1: a + b
+        functions_mapping["add_two_nums"] = add_two_nums
+        self.assertEqual(
+            parser.parse_data("${add_two_nums(1)}", functions_mapping=functions_mapping),
+            2
+        )
+        self.assertEqual(
+            parser.parse_data("${add_two_nums(1, 2)}", functions_mapping=functions_mapping),
+            3
+        )
+        self.assertEqual(
+            parser.parse_data("/api/${add_two_nums(1, 2)}", functions_mapping=functions_mapping),
+            "/api/3"
+        )
+
+        with self.assertRaises(exceptions.FunctionNotFound):
+            parser.parse_data("/api/${gen_md5(abc)}")
+
+    def test_parse_data_testcase(self):
+        variables = {
+            "uid": "1000",
+            "random": "A2dEx",
+            "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
+            "data": {"name": "user", "password": "123456"}
+        }
+        functions = {
+            "add_two_nums": lambda a, b=1: a + b,
+            "get_timestamp": lambda: int(time.time() * 1000)
+        }
+        testcase_template = {
+            "url": "http://127.0.0.1:5000/api/users/$uid/${add_two_nums(1,2)}",
+            "method": "POST",
+            "headers": {
+                "Content-Type": "application/json",
+                "authorization": "$authorization",
+                "random": "$random",
+                "sum": "${add_two_nums(1, 2)}"
+            },
+            "body": "$data"
+        }
+        parsed_testcase = parser.parse_data(testcase_template, variables, functions)
+        self.assertEqual(
+            parsed_testcase["url"],
+            "http://127.0.0.1:5000/api/users/1000/3"
+        )
+        self.assertEqual(
+            parsed_testcase["headers"]["authorization"],
+            variables["authorization"]
+        )
+        self.assertEqual(
+            parsed_testcase["headers"]["random"],
+            variables["random"]
+        )
+        self.assertEqual(
+            parsed_testcase["body"],
+            variables["data"]
+        )
+        self.assertEqual(
+            parsed_testcase["headers"]["sum"],
+            3
+        )
+
+    def test_substitute_variables(self):
+        content = {
+            'request': {
+                'url': '/api/users/$uid',
+                'headers': {'token': '$token'}
+            }
+        }
+        variables_mapping = {"$uid": 1000}
+        substituted_data = parser.substitute_variables(content, variables_mapping)
+        self.assertEqual(substituted_data["request"]["url"], "/api/users/1000")
+        self.assertEqual(substituted_data["request"]["headers"], {'token': '$token'})