refactor variable context:

1, variable context has two level, testset and testcase;
2, testset level variables can be used in whole test suite, while testcase level variables can only be used in testcase;
3, when variable binds with functions, the funtions will be called and the result will be set to the variable.

ate/__init__.py

@@ -1 +1 @@
-__version__ = '0.1.0'
+__version__ = '0.2.0'

ate/context.py

@@ -21,10 +21,10 @@ class Context(object):
     """
     def __init__(self):
         self.testset_config = {}
-        self.testset_shared_variables_mapping = dict()
+        self.testset_shared_variables_mapping = OrderedDict()
 
         self.testcase_config = {}
-        self.testcase_variables_mapping = dict()
+        self.testcase_variables_mapping = OrderedDict()
         self.init_context()
 
     def init_context(self, level='testset'):
@@ -34,12 +34,12 @@ class Context(object):
         """
         if level == "testset":
             self.testset_config["functions"] = {}
-            self.testset_config["variables"] = OrderedDict()
             self.testset_config["request"] = {}
             self.testset_shared_variables_mapping = {}
 
-        self.testcase_config["functions"] = {}
+        # testcase config shall inherit from testset configs,
-        self.testcase_config["variables"] = OrderedDict()
+        # but can not change testset configs, that's why we use copy.deepcopy here.
+        self.testcase_config["functions"] = copy.deepcopy(self.testset_config["functions"])
         self.testcase_config["request"] = {}
         self.testcase_variables_mapping = copy.deepcopy(self.testset_shared_variables_mapping)
 
@@ -64,7 +64,7 @@ class Context(object):
                 function = eval(function)
             eval_function_binds[func_name] = function
 
-        self.__update_context_config(level, "functions", eval_function_binds)
+        self.__update_context_functions_config(level, eval_function_binds)
 
     def import_module_functions(self, modules, level="testcase"):
         """ import modules and bind all functions within the context
@@ -73,11 +73,14 @@ class Context(object):
         for module_name in modules:
             imported = importlib.import_module(module_name)
             imported_functions_dict = dict(filter(is_function, vars(imported).items()))
-            self.__update_context_config(level, "functions", imported_functions_dict)
+            self.__update_context_functions_config(level, imported_functions_dict)
 
-    def register_variables_config(self, variable_binds, level="testcase"):
+    def bind_variables(self, variable_binds, level="testcase"):
-        """ register variable configs
+        """ bind variables to testset context or current testcase context.
-        @param (list) variable_binds, variable can be value or custom function
+            variables in testset context can be used in all testcases of current test suite.
+
+        @param (list) variable_binds, variable can be value or custom function.
+            if value is function, it will be called and bind result to variable.
         e.g.
             [
                 {"TOKEN": "debugtalk"},
@@ -86,71 +89,56 @@ class Context(object):
                 {"md5": "${gen_md5($TOKEN, $json, $random)}"}
             ]
         """
-        if level == "testset":
+        for variable_bind in variable_binds:
-            for variable_bind in variable_binds:
+            for variable_name, value in variable_bind.items():
-                self.testset_config["variables"].update(variable_bind)
+                variable_evale_value = self.get_eval_value(value)
-        elif level == "testcase":
-            self.testcase_config["variables"] = copy.deepcopy(self.testset_config["variables"])
-            for variable_bind in variable_binds:
-                self.testcase_config["variables"].update(variable_bind)
 
-    def register_request(self, request_dict, level="testcase"):
+                if level == "testset":
-        self.__update_context_config(level, "request", request_dict)
+                    self.testset_shared_variables_mapping[variable_name] = variable_evale_value
 
-    def __update_context_config(self, level, config_type, config_mapping):
+                self.testcase_variables_mapping[variable_name] = variable_evale_value
+
+    def __update_context_functions_config(self, level, config_mapping):
         """
         @param level: testset or testcase
-        @param config_type: functions, variables or request
+        @param config_type: functions
-        @param config_mapping: functions config mapping or variables config mapping
+        @param config_mapping: functions config mapping
         """
         if level == "testset":
-            self.testset_config[config_type].update(config_mapping)
+            self.testset_config["functions"].update(config_mapping)
-        elif level == "testcase":
+
-            self.testcase_config[config_type].update(config_mapping)
+        self.testcase_config["functions"].update(config_mapping)
+
+    def register_request(self, request_dict, level="testcase"):
+        self.__update_context_request_config(level, request_dict)
+
+    def __update_context_request_config(self, level, config_mapping):
+        """
+        @param level: testset or testcase
+        @param config_type: request
+        @param config_mapping: request config mapping
+        """
+        if level == "testset":
+            self.testset_config["request"].update(config_mapping)
+
+        self.testcase_config["request"] = utils.deep_update_dict(
+            copy.deepcopy(self.testset_config["request"]),
+            config_mapping
+        )
 
     def get_parsed_request(self):
         """ get parsed request, with each variable replaced by bind value.
-            testcase request shall inherit from testset request configs,
-            but can not change testset configs, that's why we use copy.deepcopy here.
         """
-        testcase_request_config = utils.deep_update_dict(
-            copy.deepcopy(self.testset_config["request"]),
-            self.testcase_config["request"]
-        )
-
         parsed_request = testcase.parse_template(
-            testcase_request_config,
+            self.testcase_config["request"],
-            self._get_evaluated_testcase_variables()
+            self.testcase_variables_mapping
         )
 
         return parsed_request
 
-    def bind_extracted_variables(self, variables_mapping):
-        """ bind extracted variable to current testcase context and testset context.
-            since extracted variable maybe used in current testcase and next testcases.
-        """
-        self.testset_shared_variables_mapping.update(variables_mapping)
-        self.testcase_variables_mapping.update(variables_mapping)
-
     def get_testcase_variables_mapping(self):
         return self.testcase_variables_mapping
 
-    def _get_evaluated_testcase_variables(self):
-        """ variables in variables_config will be evaluated each time
-        """
-        testcase_functions_config = copy.deepcopy(self.testset_config["functions"])
-        testcase_functions_config.update(self.testcase_config["functions"])
-        self.testcase_config["functions"] = testcase_functions_config
-
-        testcase_variables_config = copy.deepcopy(self.testset_config["variables"])
-        testcase_variables_config.update(self.testcase_config["variables"])
-        self.testcase_config["variables"] = testcase_variables_config
-
-        for var_name, var_value in self.testcase_config["variables"].items():
-            self.testcase_variables_mapping[var_name] = self.get_eval_value(var_value)
-
-        return self.testcase_variables_mapping
-
     def get_eval_value(self, data):
         """ evaluate data recursively, each variable in data will be evaluated.
         """

ate/response.py

@@ -54,23 +54,27 @@ class ResponseObject(object):
 
     def extract_response(self, extract_binds):
         """ extract content from requests.Response
-        @param (dict) extract_binds
+        @param (list) extract_binds
-            {
+            [
-                "resp_status_code": "status_code",
+                {"resp_status_code": "status_code"},
-                "resp_headers_content_type": "headers.content-type",
+                {"resp_headers_content_type": "headers.content-type"},
-                "resp_content": "content",
+                {"resp_content": "content"},
-                "resp_content_person_first_name": "content.person.name.first_name"
+                {"resp_content_person_first_name": "content.person.name.first_name"}
-            }
+            ]
+        @return (list) variable binds list
         """
-        extracted_variables_mapping = {}
+        extracted_variables_mapping_list = []
 
-        for key, field in extract_binds.items():
+        for extract_bind in extract_binds:
-            if not isinstance(field, utils.string_type):
+            for key, field in extract_bind.items():
-                raise exception.ParamsError("invalid extract_binds!")
+                if not isinstance(field, utils.string_type):
+                    raise exception.ParamsError("invalid extract_binds!")
 
-            extracted_variables_mapping[key] = self.extract_field(field)
+                extracted_variables_mapping_list.append(
+                    {key: self.extract_field(field)}
+                )
 
-        return extracted_variables_mapping
+        return extracted_variables_mapping_list
 
     def validate(self, validators, variables_mapping):
         """ Bind named validators to value within the context.

ate/runner.py

@@ -45,7 +45,7 @@ class Runner(object):
         self.context.import_module_functions(module_functions, level)
 
         variable_binds = config_dict.get('variable_binds', [])
-        self.context.register_variables_config(variable_binds, level)
+        self.context.bind_variables(variable_binds, level)
 
         request_config = config_dict.get('request', {})
         if level == "testset":
@@ -93,8 +93,8 @@ class Runner(object):
         resp_obj = response.ResponseObject(resp)
 
         extract_binds = testcase.get("extract_binds", {})
-        extracted_variables_mapping = resp_obj.extract_response(extract_binds)
+        extracted_variables_mapping_list = resp_obj.extract_response(extract_binds)
-        self.context.bind_extracted_variables(extracted_variables_mapping)
+        self.context.bind_variables(extracted_variables_mapping_list, level="testset")
 
         validators = testcase.get("validators", [])
         diff_content_list = resp_obj.validate(

tests/data/demo_binds.yml

@@ -1,10 +1,4 @@
-register_variables:
+bind_variables:
-    variable_binds:
-        - TOKEN: "debugtalk"
-        - var: [1, 2, 3]
-        - data: {'name': 'user', 'password': '123456'}
-
-register_template_variables:
     variable_binds:
         - TOKEN: "debugtalk"
         - token: $TOKEN

tests/data/demo_import_functions.yml

@@ -4,14 +4,13 @@
         - tests.data.custom_functions
     variable_binds:
         - TOKEN: debugtalk
-        - json: {}
-        - random: ${gen_random_string(5)}
-        - authorization: ${gen_md5($TOKEN, $json, $random)}
 
 - test:
     name: create user which does not exist
     variable_binds:
         - json: {"name": "user", "password": "123456"}
+        - random: ${gen_random_string(5)}
+        - authorization: ${gen_md5($TOKEN, $json, $random)}
     request:
         url: http://127.0.0.1:5000/api/users/1000
         method: POST
@@ -28,6 +27,8 @@
     name: create user which does not exist
     variable_binds:
         - json: {"name": "user", "password": "123456"}
+        - random: ${gen_random_string(5)}
+        - authorization: ${gen_md5($TOKEN, $json, $random)}
     request:
         url: http://127.0.0.1:5000/api/users/1000
         method: POST

tests/data/demo_template_sets.yml

@@ -9,9 +9,6 @@
         gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
     variable_binds:
         - TOKEN: debugtalk
-        - data: ""
-        - random: ${gen_random_string(5)}
-        - authorization: ${gen_md5($TOKEN, $data, $random)}
     request:
         base_url: http://127.0.0.1:5000
 
@@ -19,6 +16,8 @@
     name: create user which does not exist
     variable_binds:
         - data: '{"name": "user", "password": "123456"}'
+        - random: ${gen_random_string(5)}
+        - authorization: ${gen_md5($TOKEN, $data, $random)}
     request:
         url: /api/users/1000
         method: POST
@@ -35,6 +34,8 @@
     name: create user which does exist
     variable_binds:
         - data: '{"name": "user", "password": "123456"}'
+        - random: ${gen_random_string(5)}
+        - authorization: ${gen_md5($TOKEN, $data, $random)}
         - expected_status_code: 500
     request:
         url: /api/users/1000

tests/test_context.py

@@ -12,51 +12,51 @@ class VariableBindsUnittest(unittest.TestCase):
         testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml')
         self.testcases = utils.load_testcases(testcase_file_path)
 
-    def test_context_register_variables(self):
+    def test_context_bind_testset_variables(self):
         # testcase in JSON format
-        testcase1 = {
-            "variable_binds": [
-                {"TOKEN": "debugtalk"},
-                {"var": [1, 2, 3]},
-                {"data": {'name': 'user', 'password': '123456'}}
-            ]
-        }
-        # testcase in YAML format
-        testcase2 = self.testcases["register_variables"]
-
-        for testcase in [testcase1, testcase2]:
-            variable_binds = testcase['variable_binds']
-            self.context.register_variables_config(variable_binds)
-
-            context_variables = self.context._get_evaluated_testcase_variables()
-            self.assertIn("TOKEN", context_variables)
-            self.assertEqual(context_variables["TOKEN"], "debugtalk")
-            self.assertIn("var", context_variables)
-            self.assertEqual(context_variables["var"], [1, 2, 3])
-            self.assertIn("data", context_variables)
-            self.assertEqual(
-                context_variables["data"],
-                {'name': 'user', 'password': '123456'}
-            )
-
-    def test_context_register_template_variables(self):
         testcase1 = {
             "variable_binds": [
                 {"GLOBAL_TOKEN": "debugtalk"},
                 {"token": "$GLOBAL_TOKEN"}
             ]
         }
-        testcase2 = self.testcases["register_template_variables"]
+        # testcase in YAML format
+        testcase2 = self.testcases["bind_variables"]
 
         for testcase in [testcase1, testcase2]:
             variable_binds = testcase['variable_binds']
-            self.context.register_variables_config(variable_binds)
+            self.context.bind_variables(variable_binds, level="testset")
 
-            context_variables = self.context._get_evaluated_testcase_variables()
+            testset_variables = self.context.testset_shared_variables_mapping
-            self.assertIn("GLOBAL_TOKEN", context_variables)
+            testcase_variables = self.context.get_testcase_variables_mapping()
-            self.assertEqual(context_variables["GLOBAL_TOKEN"], "debugtalk")
+            self.assertIn("GLOBAL_TOKEN", testset_variables)
-            self.assertIn("token", context_variables)
+            self.assertIn("GLOBAL_TOKEN", testcase_variables)
-            self.assertEqual(context_variables["token"], "debugtalk")
+            self.assertEqual(testset_variables["GLOBAL_TOKEN"], "debugtalk")
+            self.assertIn("token", testset_variables)
+            self.assertIn("token", testcase_variables)
+            self.assertEqual(testset_variables["token"], "debugtalk")
+
+    def test_context_bind_testcase_variables(self):
+        testcase1 = {
+            "variable_binds": [
+                {"GLOBAL_TOKEN": "debugtalk"},
+                {"token": "$GLOBAL_TOKEN"}
+            ]
+        }
+        testcase2 = self.testcases["bind_variables"]
+
+        for testcase in [testcase1, testcase2]:
+            variable_binds = testcase['variable_binds']
+            self.context.bind_variables(variable_binds)
+
+            testset_variables = self.context.testset_shared_variables_mapping
+            testcase_variables = self.context.get_testcase_variables_mapping()
+            self.assertNotIn("GLOBAL_TOKEN", testset_variables)
+            self.assertIn("GLOBAL_TOKEN", testcase_variables)
+            self.assertEqual(testcase_variables["GLOBAL_TOKEN"], "debugtalk")
+            self.assertNotIn("token", testset_variables)
+            self.assertIn("token", testcase_variables)
+            self.assertEqual(testcase_variables["token"], "debugtalk")
 
     def test_context_bind_lambda_functions(self):
         testcase1 = {
@@ -76,9 +76,9 @@ class VariableBindsUnittest(unittest.TestCase):
             self.context.bind_functions(function_binds)
 
             variable_binds = testcase['variable_binds']
-            self.context.register_variables_config(variable_binds)
+            self.context.bind_variables(variable_binds)
 
-            context_variables = self.context._get_evaluated_testcase_variables()
+            context_variables = self.context.get_testcase_variables_mapping()
             self.assertIn("add1", context_variables)
             self.assertEqual(context_variables["add1"], 3)
             self.assertIn("sum2nums", context_variables)
@@ -108,8 +108,8 @@ class VariableBindsUnittest(unittest.TestCase):
             self.context.bind_functions(function_binds)
 
             variable_binds = testcase['variable_binds']
-            self.context.register_variables_config(variable_binds)
+            self.context.bind_variables(variable_binds)
-            context_variables = self.context._get_evaluated_testcase_variables()
+            context_variables = self.context.get_testcase_variables_mapping()
 
             self.assertIn("TOKEN", context_variables)
             TOKEN = context_variables["TOKEN"]
@@ -142,8 +142,8 @@ class VariableBindsUnittest(unittest.TestCase):
             self.context.import_module_functions(module_functions)
 
             variable_binds = testcase['variable_binds']
-            self.context.register_variables_config(variable_binds)
+            self.context.bind_variables(variable_binds)
-            context_variables = self.context._get_evaluated_testcase_variables()
+            context_variables = self.context.get_testcase_variables_mapping()
 
             self.assertIn("TOKEN", context_variables)
             TOKEN = context_variables["TOKEN"]

tests/test_response.py

@@ -49,40 +49,44 @@ class TestResponse(ApiServerUnittest):
             }
         )
 
-        extract_binds = {
+        extract_binds_list = [
-            "resp_status_code": "status_code",
+            {"resp_status_code": "status_code"},
-            "resp_headers_content_type": "headers.content-type",
+            {"resp_headers_content_type": "headers.content-type"},
-            "resp_content_body_success": "body.success",
+            {"resp_content_body_success": "body.success"},
-            "resp_content_content_success": "content.success",
+            {"resp_content_content_success": "content.success"},
-            "resp_content_text_success": "text.success",
+            {"resp_content_text_success": "text.success"},
-            "resp_content_person_first_name": "content.person.name.first_name",
+            {"resp_content_person_first_name": "content.person.name.first_name"},
-            "resp_content_cities_1": "content.person.cities.1"
+            {"resp_content_cities_1": "content.person.cities.1"}
-        }
+        ]
         resp_obj = response.ResponseObject(resp)
-        extract_binds_dict = resp_obj.extract_response(extract_binds)
+        extract_binds_dict_list = resp_obj.extract_response(extract_binds_list)
 
         self.assertEqual(
-            extract_binds_dict["resp_status_code"],
+            extract_binds_dict_list[0]["resp_status_code"],
             200
         )
         self.assertEqual(
-            extract_binds_dict["resp_headers_content_type"],
+            extract_binds_dict_list[1]["resp_headers_content_type"],
             "application/json"
         )
         self.assertEqual(
-            extract_binds_dict["resp_content_content_success"],
+            extract_binds_dict_list[2]["resp_content_body_success"],
             False
         )
         self.assertEqual(
-            extract_binds_dict["resp_content_text_success"],
+            extract_binds_dict_list[3]["resp_content_content_success"],
             False
         )
         self.assertEqual(
-            extract_binds_dict["resp_content_person_first_name"],
+            extract_binds_dict_list[4]["resp_content_text_success"],
+            False
+        )
+        self.assertEqual(
+            extract_binds_dict_list[5]["resp_content_person_first_name"],
             "Leo"
         )
         self.assertEqual(
-            extract_binds_dict["resp_content_cities_1"],
+            extract_binds_dict_list[6]["resp_content_cities_1"],
             "Shenzhen"
         )
 
@@ -107,21 +111,21 @@ class TestResponse(ApiServerUnittest):
             }
         )
 
-        extract_binds = {
+        extract_binds_list = [
-            "resp_content_dict_key_error": "content.not_exist"
+            {"resp_content_dict_key_error": "content.not_exist"}
-        }
+        ]
         resp_obj = response.ResponseObject(resp)
 
         with self.assertRaises(exception.ParamsError):
-            resp_obj.extract_response(extract_binds)
+            resp_obj.extract_response(extract_binds_list)
 
-        extract_binds = {
+        extract_binds_list = [
-            "resp_content_list_index_error": "content.person.cities.3"
+            {"resp_content_list_index_error": "content.person.cities.3"}
-        }
+        ]
         resp_obj = response.ResponseObject(resp)
 
         with self.assertRaises(exception.ParamsError):
-            resp_obj.extract_response(extract_binds)
+            resp_obj.extract_response(extract_binds_list)
 
     def test_extract_response_json_string(self):
         resp = requests.post(
@@ -134,14 +138,14 @@ class TestResponse(ApiServerUnittest):
             }
         )
 
-        extract_binds = {
+        extract_binds_list = [
-            "resp_content_body": "content"
+            {"resp_content_body": "content"}
-        }
+        ]
         resp_obj = response.ResponseObject(resp)
 
-        extract_binds_dict = resp_obj.extract_response(extract_binds)
+        extract_binds_dict_list = resp_obj.extract_response(extract_binds_list)
         self.assertEqual(
-            extract_binds_dict["resp_content_body"],
+            extract_binds_dict_list[0]["resp_content_body"],
             "abc"
         )
 

tests/test_runner.py

@@ -41,11 +41,11 @@ class TestRunner(ApiServerUnittest):
                     "password": "123456"
                 }
             },
-            "extract_binds": {
+            "extract_binds": [
-                "resp_status_code": "status_code",
+                {"resp_status_code": "status_code"},
-                "resp_body_success": "content.success",
+                {"resp_body_success": "content.success"},
-                "resp_headers_contenttype": "headers.content-type"
+                {"resp_headers_contenttype": "headers.content-type"}
-            },
+            ],
             "validators": [
                 {"check": "resp_status_code", "comparator": "eq", "expected": 200},
                 {"check": "resp_body_success", "comparator": "eq", "expected": False},