refactor: add extract_binds and validators

ate/context.py

@@ -8,7 +8,6 @@ class Context(object):
     def __init__(self):
         self.functions = dict()
         self.variables = dict()  # Maps variable name to value
-        self.extractors = dict()
 
     def import_requires(self, modules):
         """ import required modules dynamicly
@@ -45,19 +44,10 @@ class Context(object):
             for var_name, var_value in variable_bind_map.items():
                 self.variables[var_name] = self.get_eval_value(var_value)
 
-    def bind_extractors(self, extract_binds):
+    def update_variables(self, variables_mapping):
-        """ Bind named extractors to value within the context.
+        """ update context variables binds with new variables mapping
-            value => parsed from requests.Response object
-            key => extractor name, can be used as variable in next testcases
-        @param (dict) extract_binds
-            {
-                "resp_status_code": "status_code",
-                "resp_headers": "headers",
-                "resp_headers_content_type": "headers.content-type",
-                "resp_content": "content"
-            }
         """
-        self.extractors.update(extract_binds)
+        self.variables.update(variables_mapping)
 
     def get_eval_value(self, data):
         """ evaluate data recursively, each variable in data will be evaluated.

ate/main.py

@@ -26,7 +26,7 @@ def create_suite(testset):
 
     test_runner = runner.TestRunner()
     config_dict = testset.get("config", {})
-    test_runner.pre_config(config_dict)
+    test_runner.update_context(config_dict)
     testcases = testset.get("testcases", [])
 
     for testcase in testcases:

ate/response.py

@@ -1,16 +1,18 @@
 from ate import utils, exception
 
+try:
+    basestring  # Python 2.x
+except NameError:
+    basestring = str    # Python 3.x
 
 class ResponseObject(object):
 
-    def __init__(self, resp_obj, context=None):
+    def __init__(self, resp_obj):
         """ initialize with a requests.Response object
         @param (requests.Response instance) resp_obj
-        @param (ate.context.Context instance) context
         """
         self.resp_obj = resp_obj
-        if context:
+        self.success = True
-            self.extract_response(context)
 
     def parsed_body(self):
         try:
@@ -25,49 +27,9 @@ class ResponseObject(object):
             'body': self.parsed_body()
         }
 
-    def diff_response(self, expected_resp_json):
+    def extract_response(self, extract_binds, delimiter='.'):
-        diff_content = {}
+        """ extract content from requests.Response
-        resp_info = self.parsed_dict()
+        @param (dict) extract_binds
-
-        expected_status_code = expected_resp_json.get('status_code', 200)
-        if resp_info['status_code'] != int(expected_status_code):
-            diff_content['status_code'] = {
-                'value': resp_info['status_code'],
-                'expected': expected_status_code
-            }
-
-        expected_headers = expected_resp_json.get('headers', {})
-        headers_diff = utils.diff_json(resp_info['headers'], expected_headers)
-        if headers_diff:
-            diff_content['headers'] = headers_diff
-
-        expected_body = expected_resp_json.get('body', None)
-
-        if expected_body is None:
-            body_diff = {}
-        elif type(expected_body) != type(resp_info['body']):
-            body_diff = {
-                'value': resp_info['body'],
-                'expected': expected_body
-            }
-        elif isinstance(expected_body, str):
-            if expected_body != resp_info['body']:
-                body_diff = {
-                    'value': resp_info['body'],
-                    'expected': expected_body
-                }
-        elif isinstance(expected_body, dict):
-            body_diff = utils.diff_json(resp_info['body'], expected_body)
-
-        if body_diff:
-            diff_content['body'] = body_diff
-
-        return diff_content
-
-    def extract_response(self, context, delimiter='.'):
-        """ extract content from requests.Response, and bind extracted value to context.extractors
-        @param (ate.context.Context instance) context
-            context.extractors:
             {
                 "resp_status_code": "status_code",
                 "resp_headers_content_type": "headers.content-type",
@@ -75,34 +37,73 @@ class ResponseObject(object):
                 "resp_content_person_first_name": "content.person.name.first_name"
             }
         """
-        for key, value in context.extractors.items():
+        extract_binds_dict = {}
+
+        for key, value in extract_binds.items():
+            if not isinstance(value, basestring):
+                raise exception.ParamsError("invalid extract_binds!")
+
             try:
-                if isinstance(value, str):
+                value += "."
-                    value += "."
+                # string.split(sep=None, maxsplit=-1) -> list of strings
-                    # string.split(sep=None, maxsplit=-1) -> list of strings
+                # e.g. "content.person.name" => ["content", "person.name"]
-                    # e.g. "content.person.name" => ["content", "person.name"]
+                top_query, sub_query = value.split(delimiter, 1)
-                    top_query, sub_query = value.split(delimiter, 1)
-
-                    if top_query in ["body", "content", "text"]:
-                        json_content = self.parsed_body()
-                    else:
-                        json_content = getattr(self.resp_obj, top_query)
-
-                    if sub_query:
-                        # e.g. key: resp_headers_content_type, sub_query = "content-type"
-                        answer = utils.query_json(json_content, sub_query)
-                        context.extractors[key] = answer
-                    else:
-                        # e.g. key: resp_status_code, resp_content
-                        context.extractors[key] = json_content
 
+                if top_query in ["body", "content", "text"]:
+                    json_content = self.parsed_body()
                 else:
-                    raise NotImplementedError("TODO: support template.")
+                    json_content = getattr(self.resp_obj, top_query)
+
+                if sub_query:
+                    # e.g. key: resp_headers_content_type, sub_query = "content-type"
+                    answer = utils.query_json(json_content, sub_query)
+                    extract_binds_dict[key] = answer
+                else:
+                    # e.g. key: resp_status_code, resp_content
+                    extract_binds_dict[key] = json_content
 
             except AttributeError:
                 raise exception.ParamsError("invalid extract_binds!")
 
-    def validate(self, expected_resp_json):
+        return extract_binds_dict
-        diff_content = self.diff_response(expected_resp_json)
+
-        success = False if diff_content else True
+    def validate(self, validators, variables_mapping):
-        return success, diff_content
+        """ Bind named validators to value within the context.
+        @param (dict) validators
+            {
+                "resp_status_code": {"comparator": "eq", "expected": 201},
+                "resp_body_success": {"comparator": "eq", "expected": True}
+            }
+        @param (dict) variables_mapping
+            {
+                "resp_status_code": 200,
+                "resp_body_success": True
+            }
+        @return (dict) content differences
+            {
+                "resp_status_code": {
+                    "comparator": "eq", "expected": 201, "value": 200
+                }
+            }
+        """
+        diff_content_dict = {}
+
+        for validator_key, validator_dict in validators.items():
+
+            try:
+                value = variables_mapping[validator_key]
+                validator_dict["value"] = value
+            except KeyError:
+                raise exception.ParamsError("invalid validator %s" % validator_key)
+
+            difference_exist = utils.compare(
+                value,
+                validator_dict["expected"],
+                validator_dict["comparator"]
+            )
+
+            if difference_exist:
+                diff_content_dict[validator_key] = validator_dict
+
+        self.success = False if diff_content_dict else True
+        return diff_content_dict

ate/runner.py

@@ -2,7 +2,7 @@ import requests
 
 from ate import exception, response
 from ate.context import Context
-from ate.testcase import TestcaseParser
+from ate.testcase import parse_template
 
 
 class TestRunner(object):
@@ -10,10 +10,9 @@ class TestRunner(object):
     def __init__(self):
         self.client = requests.Session()
         self.context = Context()
-        self.testcase_parser = TestcaseParser()
 
-    def pre_config(self, config_dict):
+    def update_context(self, config_dict):
-        """ create/update variables binds
+        """ create/update context variables binds
         @param config_dict
             {
                 "name": "description content",
@@ -41,44 +40,6 @@ class TestRunner(object):
         variable_binds = config_dict.get('variable_binds', [])
         self.context.bind_variables(variable_binds)
 
-        extract_binds = config_dict.get('extract_binds', {})
-        self.context.bind_extractors(extract_binds)
-
-        self.testcase_parser.update_variables_binds(self.context.variables)
-
-    def parse_testcase(self, testcase):
-        """ parse testcase with variables binds if it is a template.
-        @param (dict) testcase
-            {
-                "name": "testcase description",
-                "requires": [],  # optional, override
-                "function_binds": {}, # optional, override
-                "variable_binds": {}, # optional, override
-                "request": {},
-                "response": {}
-            }
-        @return (dict) parsed testcase with bind values
-            {
-                "request": {
-                    "url": "http://127.0.0.1:5000/api/users/1000",
-                    "method": "POST",
-                    "headers": {
-                        "Content-Type": "application/json",
-                        "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
-                        "random": "A2dEx"
-                    },
-                    "body": '{"name": "user", "password": "123456"}'
-                },
-                "response": {
-                    "status_code": 201
-                }
-            }
-        """
-        self.pre_config(testcase)
-
-        parsed_testcase = self.testcase_parser.parse(testcase)
-        return parsed_testcase
-
     def run_test(self, testcase):
         """ run single testcase.
         @param (dict) testcase
@@ -87,26 +48,42 @@ class TestRunner(object):
                 "requires": [],  # optional, override
                 "function_binds": {}, # optional, override
                 "variable_binds": {}, # optional, override
-                "request": {},
+                "request": {
-                "response": {}
+                    "url": "http://127.0.0.1:5000/api/users/1000",
+                    "method": "POST",
+                    "headers": {
+                        "Content-Type": "application/json",
+                        "authorization": "${authorization}",
+                        "random": "${random}"
+                    },
+                    "body": '{"name": "user", "password": "123456"}'
+                },
+                "extract_binds": {},
+                "validators": {}
             }
         @return (tuple) test result of single testcase
             (success, diff_content)
         """
-        testcase = self.parse_testcase(testcase)
+        self.update_context(testcase)
-
+        parsed_request = parse_template(testcase["request"], self.context.variables)
-        req_kwargs = testcase['request']
 
         try:
-            url = req_kwargs.pop('url')
+            url = parsed_request.pop('url')
-            method = req_kwargs.pop('method')
+            method = parsed_request.pop('method')
         except KeyError:
             raise exception.ParamsError("URL or METHOD missed!")
 
-        resp = self.client.request(url=url, method=method, **req_kwargs)
+        resp = self.client.request(url=url, method=method, **parsed_request)
+        resp_obj = response.ResponseObject(resp)
 
-        resp_obj = response.ResponseObject(resp, self.context)
+        extract_binds = testcase.get("extract_binds", {})
-        return resp_obj.validate(testcase['response'])
+        extract_binds_dict = resp_obj.extract_response(extract_binds)
+        self.context.update_variables(extract_binds_dict)
+
+        validators = testcase.get("validators", {})
+        diff_content_dict = resp_obj.validate(validators, self.context.variables)
+
+        return resp_obj.success, diff_content_dict
 
     def run_testset(self, testset):
         """ run single testset, including one or several testcases.
@@ -124,7 +101,8 @@ class TestRunner(object):
                         "name": "testcase description",
                         "variable_binds": {}, # override
                         "request": {},
-                        "response": {}
+                        "extract_binds": {},
+                        "validators": {}
                     },
                     testcase12
                 ]
@@ -138,7 +116,7 @@ class TestRunner(object):
         results = []
 
         config_dict = testset.get("config", {})
-        self.pre_config(config_dict)
+        self.update_context(config_dict)
         testcases = testset.get("testcases", [])
         for testcase in testcases:
             result = self.run_test(testcase)

ate/testcase.py

@@ -1,69 +1,56 @@
 from ate import utils
 
-class TestcaseParser(object):
 
-    def __init__(self, variables_binds={}):
+def parse_template(testcase_template, variables_binds):
-        self.variables_binds = variables_binds
+    """ parse testcase_template, replace all variables with bind value.
+    variables marker: ${variable}.
+    @param (dict) testcase_template
+        {
+            "url": "http://127.0.0.1:5000/api/users/${uid}",
+            "method": "POST",
+            "headers": {
+                "Content-Type": "application/json",
+                "authorization": "${authorization}",
+                "random": "${random}"
+            },
+            "body": "${data}"
+        }
+    @param (dict) variables binds mapping
+        {
+            "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
+            "random": "A2dEx",
+            "data": '{"name": "user", "password": "123456"}'
+        }
+    @return (dict) parsed testcase with bind variable values
+        {
+            "url": "http://127.0.0.1:5000/api/users/1000",
+            "method": "POST",
+            "headers": {
+                "Content-Type": "application/json",
+                "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
+                "random": "A2dEx"
+            },
+            "body": '{"name": "user", "password": "123456"}'
+        }
+    """
 
-    def update_variables_binds(self, variables_mapping):
+    def substitute(content):
-        """ update variables binds with new mapping.
-        """
-        if variables_mapping:
-            self.variables_binds.update(variables_mapping)
-
-    def parse(self, testcase_template):
-        """ parse testcase_template, replace all variables with bind value.
-        variables marker: ${variable}.
-        @param (dict) testcase_template
-            {
-                "request": {
-                    "url": "http://127.0.0.1:5000/api/users/${uid}",
-                    "method": "POST",
-                    "headers": {
-                        "Content-Type": "application/json",
-                        "authorization": "${authorization}",
-                        "random": "${random}"
-                    },
-                    "body": "${data}"
-                },
-                "response": {
-                    "status_code": "${expected_status}"
-                }
-            }
-        @return (dict) parsed testcase with bind values
-            {
-                "request": {
-                    "url": "http://127.0.0.1:5000/api/users/1000",
-                    "method": "POST",
-                    "headers": {
-                        "Content-Type": "application/json",
-                        "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
-                        "random": "A2dEx"
-                    },
-                    "body": '{"name": "user", "password": "123456"}'
-                },
-                "response": {
-                    "status_code": 201
-                }
-            }
-        """
-        return self.substitute(testcase_template)
-
-    def substitute(self, content):
         """ substitute content recursively, each variable will be replaced with bind value.
             variables marker: ${variable}.
         """
         if isinstance(content, str):
-            return utils.parse_content_with_variables(content, self.variables_binds)
+            return utils.parse_content_with_variables(content, variables_binds)
 
         if isinstance(content, list):
-            return [self.substitute(item) for item in content]
+            return [substitute(item) for item in content]
 
         if isinstance(content, dict):
             parsed_content = {}
             for key, value in content.items():
-                parsed_content[key] = self.substitute(value)
+                parsed_content[key] = substitute(value)
 
             return parsed_content
 
         return content
+
+    return substitute(testcase_template)

ate/utils.py

@@ -62,19 +62,6 @@ def load_testcases(testcase_file_path):
         # '' or other suffix
         raise ParamsError("Bad testcase file name!")
 
-def diff_json(current_json, expected_json):
-    json_diff = {}
-
-    for key, expected_value in expected_json.items():
-        value = current_json.get(key, None)
-        if str(value) != str(expected_value):
-            json_diff[key] = {
-                'value': value,
-                'expected': expected_value
-            }
-
-    return json_diff
-
 def load_foler_files(folder_path):
     """ load folder path, return all files in list format.
     """
@@ -192,3 +179,25 @@ def query_json(json_content, query, delimiter='.'):
         raise ParamsError("invalid query string in extract_binds!")
 
     return json_content
+
+def diff_json(current_json, expected_json):
+    json_diff = {}
+
+    for key, expected_value in expected_json.items():
+        value = current_json.get(key, None)
+        if str(value) != str(expected_value):
+            json_diff[key] = {
+                'value': value,
+                'expected': expected_value
+            }
+
+    return json_diff
+
+def compare(value, expected, comparator="eq"):
+    try:
+        if comparator in ["eq", "=="]:
+            assert value == expected
+
+        return False
+    except AssertionError:
+        return True

test/data/demo_template_separate.yml

@@ -13,7 +13,6 @@
         - random: {"func": "gen_random_string", "args": [5]}
         - data: '{"name": "user", "password": "123456"}'
         - authorization: {"func": "gen_md5", "args": ["${TOKEN}", "${data}", "${random}"]}
-        - expected_status_code: 201
     request:
         url: http://127.0.0.1:5000/api/users/1000
         method: POST
@@ -22,13 +21,12 @@
             authorization: "${authorization}"
             random: "${random}"
         data: "${data}"
-    response:
+    extract_binds:
-        status_code: "${expected_status_code}"
+        resp_status_code: status_code
-        headers:
+        resp_body_success: content.success
-            Content-Type: application/json
+    validators:
-        body:
+        resp_status_code: {"comparator": "eq", "expected": 201}
-            success: true
+        resp_body_success: {"comparator": "eq", "expected": true}
-            msg: user created successfully.
 
 - test:
     name: create user which does not exist
@@ -44,7 +42,6 @@
         - random: {"func": "gen_random_string", "args": [5]}
         - data: '{"name": "user", "password": "123456"}'
         - authorization: {"func": "gen_md5", "args": ["${TOKEN}", "${data}", "${random}"]}
-        - expected_status_code: 500
     request:
         url: http://127.0.0.1:5000/api/users/1000
         method: POST
@@ -53,10 +50,9 @@
             authorization: "${authorization}"
             random: "${random}"
         data: "${data}"
-    response:
+    extract_binds:
-        status_code: "${expected_status_code}"
+        resp_status_code: status_code
-        headers:
+        resp_body_success: content.success
-            Content-Type: application/json
+    validators:
-        body:
+        resp_status_code: {"comparator": "eq", "expected": 500}
-            success: false
+        resp_body_success: {"comparator": "eq", "expected": false}
-            msg: user already existed.

test/data/demo_template_sets.yml

@@ -17,7 +17,6 @@
     name: create user which does not exist
     variable_binds:
         - data: '{"name": "user", "password": "123456"}'
-        - expected_status_code: 201
     request:
         url: http://127.0.0.1:5000/api/users/1000
         method: POST
@@ -26,13 +25,12 @@
             authorization: "${authorization}"
             random: "${random}"
         data: "${data}"
-    response:
+    extract_binds:
-        status_code: "${expected_status_code}"
+        resp_status_code: status_code
-        headers:
+        resp_body_success: content.success
-            Content-Type: application/json
+    validators:
-        body:
+        resp_status_code: {"comparator": "eq", "expected": 201}
-            success: true
+        resp_body_success: {"comparator": "eq", "expected": true}
-            msg: user created successfully.
 
 - test:
     name: create user which does not exist
@@ -47,10 +45,9 @@
             authorization: "${authorization}"
             random: "${random}"
         data: "${data}"
-    response:
+    extract_binds:
-        status_code: "${expected_status_code}"
+        resp_status_code: status_code
-        headers:
+        resp_body_success: content.success
-            Content-Type: application/json
+    validators:
-        body:
+        resp_status_code: {"comparator": "eq", "expected": 500}
-            success: false
+        resp_body_success: {"comparator": "eq", "expected": false}
-            msg: user already existed.

test/data/simple_demo_auth_hardcode.json

@@ -15,15 +15,14 @@
                     "password": "123456"
                 }
             },
-            "response": {
+            "extract_binds": {
-                "status_code": 201,
+                "resp_status_code": "status_code",
-                "headers": {
+                "resp_body_success": "content.success",
-                    "Content-Type": "application/json"
+                "resp_body_msg": "content.msg"
-                },
+            },
-                "body": {
+            "validators": {
-                    "success": true,
+                "resp_status_code": {"comparator": "eq", "expected": 201},
-                    "msg": "user created successfully."
+                "resp_body_success": {"comparator": "eq", "expected": true}
-                }
             }
         }
     },
@@ -43,15 +42,13 @@
                     "password": "123456"
                 }
             },
-            "response": {
+            "extract_binds": {
-                "status_code": 500,
+                "resp_status_code": "status_code",
-                "headers": {
+                "resp_body_success": "content.success"
-                    "Content-Type": "application/json"
+            },
-                },
+            "validators": {
-                "body":{
+                "resp_status_code": {"comparator": "eq", "expected": 500},
-                    "success": false,
+                "resp_body_success": {"comparator": "eq", "expected": false}
-                    "msg": "user already existed."
-                }
             }
         }
     }

test/data/simple_demo_auth_hardcode.yml

@@ -10,13 +10,13 @@
         json:
             name: "user1"
             password: "123456"
-    response:
+    extract_binds:
-        status_code: 201
+        resp_status_code: status_code
-        headers:
+        resp_body_success: content.success
-            Content-Type: application/json
+        resp_body_msg: content.msg
-        body:
+    validators:
-            success: true
+        resp_status_code: {"comparator": "eq", "expected": 201}
-            msg: user created successfully.
+        resp_body_success: {"comparator": "eq", "expected": true}
 
 - test:
     name: create user which existed
@@ -30,10 +30,9 @@
         json:
             name: "user1"
             password: "123456"
-    response:
+    extract_binds:
-        status_code: 500
+        resp_status_code: status_code
-        headers:
+        resp_body_success: content.success
-            Content-Type: application/json
+    validators:
-        body:
+        resp_status_code: {"comparator": "eq", "expected": 500}
-            success: false
+        resp_body_success: {"comparator": "eq", "expected": false}
-            msg: user already existed.

test/data/simple_demo_no_auth.json

@@ -14,15 +14,14 @@
                     "password": "123456"
                 }
             },
-            "response": {
+            "extract_binds": {
-                "status_code": 201,
+                "resp_status_code": "status_code",
-                "headers": {
+                "resp_body_success": "content.success",
-                    "Content-Type": "application/json"
+                "resp_body_msg": "content.msg"
-                },
+            },
-                "body": {
+            "validators": {
-                    "success": true,
+                "resp_status_code": {"comparator": "eq", "expected": 201},
-                    "msg": "user created successfully."
+                "resp_body_success": {"comparator": "eq", "expected": true}
-                }
             }
         }
     },
@@ -40,15 +39,13 @@
                     "password": "123456"
                 }
             },
-            "response": {
+            "extract_binds": {
-                "status_code": 500,
+                "resp_status_code": "status_code",
-                "headers": {
+                "resp_body_success": "content.success"
-                    "Content-Type": "application/json"
+            },
-                },
+            "validators": {
-                "body":{
+                "resp_status_code": {"comparator": "eq", "expected": 500},
-                    "success": false,
+                "resp_body_success": {"comparator": "eq", "expected": false}
-                    "msg": "user already existed."
-                }
             }
         }
     }

test/data/simple_demo_no_auth.yml

@@ -8,13 +8,14 @@
         json:
             name: user1
             password: 123456
-    response:
+    extract_binds:
-        status_code: 201
+        resp_status_code: status_code
-        headers:
+        resp_body_success: content.success
-            Content-Type: application/json
+        resp_headers_content_type: headers.content-type
-        body:
+    validators:
-            success: true
+        resp_status_code: {"comparator": "eq", "expected": 201}
-            msg: user created successfully.
+        resp_headers_content_type: {"comparator": "eq", "expected": "application/json"}
+        resp_body_success: {"comparator": "eq", "expected": true}
 
 - test:
     name: create user which existed
@@ -26,10 +27,11 @@
         json:
             name: user1
             password: 123456
-    response:
+    extract_binds:
-        status_code: 500
+        resp_status_code: status_code
-        headers:
+        resp_body_success: content.success
-            Content-Type: application/json
+        resp_headers_content_type: headers.content-type
-        body:
+    validators:
-            success: false
+        resp_status_code: {"comparator": "eq", "expected": 500}
-            msg: user already existed.
+        resp_headers_content_type: {"comparator": "eq", "expected": "application/json"}
+        resp_body_success: {"comparator": "eq", "expected": false}

test/test_response.py

@@ -1,6 +1,5 @@
-import random
 import requests
-from ate import response, context, exception
+from ate import response, exception
 from test.base import ApiServerUnittest
 
 class TestResponse(ApiServerUnittest):
@@ -29,199 +28,6 @@ class TestResponse(ApiServerUnittest):
         self.assertIn('Content-Length', parsed_dict['headers'])
         self.assertTrue(str, type(parsed_dict['body']))
 
-    def test_diff_response_status_code_equal(self):
-        status_code = random.randint(200, 511)
-        resp = requests.post(
-            url="http://127.0.0.1:5000/customize-response",
-            json={
-                'status_code': status_code,
-            }
-        )
-
-        expected_resp_json = {
-            'status_code': status_code
-        }
-        resp_obj = response.ResponseObject(resp)
-        diff_content = resp_obj.diff_response(expected_resp_json)
-        self.assertFalse(diff_content)
-
-    def test_diff_response_status_code_not_equal(self):
-        status_code = random.randint(200, 511)
-        resp = requests.post(
-            url="http://127.0.0.1:5000/customize-response",
-            json={
-                'status_code': status_code,
-            }
-        )
-
-        expected_resp_json = {
-            'status_code': 512
-        }
-        resp_obj = response.ResponseObject(resp)
-        diff_content = resp_obj.diff_response(expected_resp_json)
-        self.assertIn('value', diff_content['status_code'])
-        self.assertIn('expected', diff_content['status_code'])
-        self.assertEqual(diff_content['status_code']['value'], status_code)
-        self.assertEqual(diff_content['status_code']['expected'], 512)
-
-    def test_diff_response_headers_equal(self):
-        resp = requests.post(
-            url="http://127.0.0.1:5000/customize-response",
-            json={
-                'headers': {
-                    'abc': 123,
-                    'def': 456
-                }
-            }
-        )
-
-        expected_resp_json = {
-            'headers': {
-                'abc': 123,
-                'def': '456'
-            }
-        }
-        resp_obj = response.ResponseObject(resp)
-        diff_content = resp_obj.diff_response(expected_resp_json)
-        self.assertFalse(diff_content)
-
-    def test_diff_response_headers_not_equal(self):
-        resp = requests.post(
-            url="http://127.0.0.1:5000/customize-response",
-            json={
-                'headers': {
-                    'a': 123,
-                    'b': '456',
-                    'c': '789'
-                }
-            }
-        )
-
-        expected_resp_json = {
-            'headers': {
-                'a': '123',
-                'b': '457',
-                'd': 890
-            }
-        }
-        resp_obj = response.ResponseObject(resp)
-        diff_content = resp_obj.diff_response(expected_resp_json)
-        self.assertEqual(
-            diff_content['headers'],
-            {
-                'b': {'expected': '457', 'value': '456'},
-                'd': {'expected': 890, 'value': None}
-            }
-        )
-
-    def test_diff_response_body_equal(self):
-        resp = requests.post(
-            url="http://127.0.0.1:5000/customize-response",
-            json={
-                'body': {
-                    'success': True,
-                    'count': 10
-                }
-            }
-        )
-
-        # expected response body is not specified
-        expected_resp_json = {}
-        resp_obj = response.ResponseObject(resp)
-        diff_content = resp_obj.diff_response(expected_resp_json)
-        self.assertFalse(diff_content)
-
-        # response body is the same as expected response body
-        expected_resp_json = {
-            'body': {
-                'success': True,
-                'count': '10'
-            }
-        }
-        resp_obj = response.ResponseObject(resp)
-        diff_content = resp_obj.diff_response(expected_resp_json)
-        self.assertFalse(diff_content)
-
-    def test_diff_response_body_not_equal_type_unmatch(self):
-        resp = requests.post(
-            url="http://127.0.0.1:5000/customize-response",
-            json={
-                'body': {
-                    'success': True,
-                    'count': 10
-                }
-            }
-        )
-
-        # response body content type not match
-        expected_resp_json = {
-            'body': "ok"
-        }
-        resp_obj = response.ResponseObject(resp)
-        diff_content = resp_obj.diff_response(expected_resp_json)
-        self.assertEqual(
-            diff_content['body'],
-            {
-                'value': {'success': True, 'count': 10},
-                'expected': 'ok'
-            }
-        )
-
-    def test_diff_response_body_not_equal_string_unmatch(self):
-        resp = requests.post(
-            url="http://127.0.0.1:5000/customize-response",
-            json={
-                'body': "success"
-            }
-        )
-
-        # response body content type matched to be string, while value unmatch
-        expected_resp_json = {
-            'body': "ok"
-        }
-        resp_obj = response.ResponseObject(resp)
-        diff_content = resp_obj.diff_response(expected_resp_json)
-        self.assertEqual(
-            diff_content['body'],
-            {
-                'value': 'success',
-                'expected': 'ok'
-            }
-        )
-
-    def test_diff_response_body_not_equal_json_unmatch(self):
-        resp = requests.post(
-            url="http://127.0.0.1:5000/customize-response",
-            json={
-                'body': {
-                    'success': False
-                }
-            }
-        )
-
-        # response body is the same as expected response body
-        expected_resp_json = {
-            'body': {
-                'success': True,
-                'count': 10
-            }
-        }
-        resp_obj = response.ResponseObject(resp)
-        diff_content = resp_obj.diff_response(expected_resp_json)
-        self.assertEqual(
-            diff_content['body'],
-            {
-                'success': {
-                    'value': False,
-                    'expected': True
-                },
-                'count': {
-                    'value': None,
-                    'expected': 10
-                }
-            }
-        )
-
     def test_extract_response_json(self):
         resp = requests.post(
             url="http://127.0.0.1:5000/customize-response",
@@ -252,12 +58,9 @@ class TestResponse(ApiServerUnittest):
             "resp_content_person_first_name": "content.person.name.first_name",
             "resp_content_cities_1": "content.person.cities.1"
         }
+        resp_obj = response.ResponseObject(resp)
+        extract_binds_dict = resp_obj.extract_response(extract_binds)
 
-        test_context = context.Context()
-        test_context.bind_extractors(extract_binds)
-        response.ResponseObject(resp, test_context)
-
-        extract_binds_dict = test_context.extractors
         self.assertEqual(
             extract_binds_dict["resp_status_code"],
             200
@@ -283,7 +86,6 @@ class TestResponse(ApiServerUnittest):
             "Shenzhen"
         )
 
-
     def test_extract_response_fail(self):
         resp = requests.post(
             url="http://127.0.0.1:5000/customize-response",
@@ -308,22 +110,18 @@ class TestResponse(ApiServerUnittest):
         extract_binds = {
             "resp_content_dict_key_error": "content.not_exist"
         }
-
+        resp_obj = response.ResponseObject(resp)
-        test_context = context.Context()
-        test_context.bind_extractors(extract_binds)
 
         with self.assertRaises(exception.ParamsError):
-            response.ResponseObject(resp, test_context)
+            resp_obj.extract_response(extract_binds)
 
         extract_binds = {
             "resp_content_list_index_error": "content.person.cities.3"
         }
-
+        resp_obj = response.ResponseObject(resp)
-        test_context = context.Context()
-        test_context.bind_extractors(extract_binds)
 
         with self.assertRaises(exception.ParamsError):
-            response.ResponseObject(resp, test_context)
+            resp_obj.extract_response(extract_binds)
 
     def test_extract_response_json_string(self):
         resp = requests.post(
@@ -339,13 +137,48 @@ class TestResponse(ApiServerUnittest):
         extract_binds = {
             "resp_content_body": "content"
         }
+        resp_obj = response.ResponseObject(resp)
 
-        test_context = context.Context()
+        extract_binds_dict = resp_obj.extract_response(extract_binds)
-        test_context.bind_extractors(extract_binds)
-        resp_obj = response.ResponseObject(resp, test_context)
-
-        extract_binds_dict = test_context.extractors
         self.assertEqual(
             extract_binds_dict["resp_content_body"],
             "abc"
         )
+
+    def test_validate(self):
+        url = "http://127.0.0.1:5000/"
+        resp = requests.get(url)
+        resp_obj = response.ResponseObject(resp)
+
+        validators = {
+            "resp_status_code": {"comparator": "eq", "expected": 201},
+            "resp_body_success": {"comparator": "eq", "expected": True}
+        }
+        variables_mapping = {
+            "resp_status_code": 200,
+            "resp_body_success": True
+        }
+
+        diff_content_dict = resp_obj.validate(validators, variables_mapping)
+        self.assertFalse(resp_obj.success)
+        self.assertEqual(
+            diff_content_dict,
+            {
+                "resp_status_code": {
+                    "comparator": "eq", "expected": 201, "value": 200
+                }
+            }
+        )
+
+        validators = {
+            "resp_status_code": {"comparator": "eq", "expected": 201},
+            "resp_body_success": {"comparator": "eq", "expected": True}
+        }
+        variables_mapping = {
+            "resp_status_code": 201,
+            "resp_body_success": True
+        }
+
+        diff_content_dict = resp_obj.validate(validators, variables_mapping)
+        self.assertTrue(resp_obj.success)
+        self.assertEqual(diff_content_dict, {})

test/test_runner.py

@@ -41,39 +41,31 @@ class TestRunner(ApiServerUnittest):
                     "password": "123456"
                 }
             },
-            "response": {
+            "extract_binds": {
-                "status_code": 200,
+                "resp_status_code": "status_code",
-                "headers": {
+                "resp_body_success": "content.success",
-                    "Content-Type": "html/text"
+                "resp_headers_contenttype": "headers.content-type"
-                },
+            },
-                "body": {
+            "validators": {
-                    'success': False,
+                "resp_status_code": {"comparator": "eq", "expected": 200},
-                    'msg': "user already existed."
+                "resp_body_success": {"comparator": "eq", "expected": False},
-                }
+                "resp_headers_contenttype": {"comparator": "eq", "expected": "html/text"}
             }
         }
+
         success, diff_content = self.test_runner.run_test(testcase)
         self.assertFalse(success)
         self.assertEqual(
-            diff_content['status_code'],
+            diff_content['resp_status_code'],
-            {'expected': 200, 'value': 201}
+            {"comparator": "eq", "expected": 200, 'value': 201}
         )
         self.assertEqual(
-            diff_content['headers'],
+            diff_content['resp_body_success'],
-            {'Content-Type': {'expected': 'html/text', 'value': 'application/json'}}
+            {"comparator": "eq", "expected": False, 'value': True}
         )
         self.assertEqual(
-            diff_content['body'],
+            diff_content['resp_headers_contenttype'],
-            {
+            {"comparator": "eq", "expected": "html/text", 'value': "application/json"}
-                'msg': {
-                    'expected': 'user already existed.',
-                    'value': 'user created successfully.'
-                },
-                'success': {
-                    'expected': False,
-                    'value': True
-                }
-            }
         )
 
     def test_run_testset_json_success(self):

test/test_testcase.py

@@ -1,6 +1,6 @@
 import unittest
 
-from ate.testcase import TestcaseParser
+from ate.testcase import parse_template
 from ate import exception
 
 
@@ -18,7 +18,6 @@ class TestcaseParserUnittest(unittest.TestCase):
             "expected_status": 201,
             "expected_success": True
         }
-        self.testcase_parser = TestcaseParser(self.variables_binds)
 
     def test_parse_testcase_template(self):
         testcase = {
@@ -43,7 +42,7 @@ class TestcaseParserUnittest(unittest.TestCase):
                 }
             }
         }
-        parsed_testcase = self.testcase_parser.parse(testcase)
+        parsed_testcase = parse_template(testcase, self.variables_binds)
 
         self.assertEqual(
             parsed_testcase["request"]["url"],
@@ -78,7 +77,7 @@ class TestcaseParserUnittest(unittest.TestCase):
             }
         }
         with self.assertRaises(exception.ParamsError):
-            self.testcase_parser.parse(testcase)
+            parse_template(testcase, self.variables_binds)
 
     def test_parse_testcase_with_new_variable_binds(self):
         testcase = {
@@ -90,10 +89,9 @@ class TestcaseParserUnittest(unittest.TestCase):
         new_variable_binds = {
             "method": "GET"
         }
-        self.testcase_parser.update_variables_binds(new_variable_binds)
+        self.variables_binds.update(new_variable_binds)
-        parsed_testcase = self.testcase_parser.parse(testcase)
+        parsed_testcase = parse_template(testcase, self.variables_binds)
 
-        self.assertIn("method", self.testcase_parser.variables_binds)
         self.assertEqual(
             parsed_testcase["request"]["method"],
             new_variable_binds["method"]

test/test_utils.py

@@ -18,7 +18,6 @@ class TestUtils(ApiServerUnittest):
         testcase = testcases[0]["test"]
         self.assertIn('name', testcase)
         self.assertIn('request', testcase)
-        self.assertIn('response', testcase)
         self.assertIn('url', testcase['request'])
         self.assertIn('method', testcase['request'])
 
@@ -30,7 +29,6 @@ class TestUtils(ApiServerUnittest):
         testcase = testcases[0]["test"]
         self.assertIn('name', testcase)
         self.assertIn('request', testcase)
-        self.assertIn('response', testcase)
         self.assertIn('url', testcase['request'])
         self.assertIn('method', testcase['request'])
 
@@ -76,7 +74,6 @@ class TestUtils(ApiServerUnittest):
             for testcase in testset["testcases"]:
                 self.assertIn('name', testcase)
                 self.assertIn('request', testcase)
-                self.assertIn('response', testcase)
                 self.assertIn('url', testcase['request'])
                 self.assertIn('method', testcase['request'])
 
@@ -179,3 +176,11 @@ class TestUtils(ApiServerUnittest):
         query = "person.name.first_name"
         result = utils.query_json(json_content, query)
         self.assertEqual(result, "Leo")
+
+    def test_compare(self):
+        self.assertEqual(utils.compare(1, 1, "eq"), False)
+        self.assertEqual(utils.compare("abc", "abc", "eq"), False)
+        self.assertEqual(utils.compare("abc", "abc"), False)
+
+        self.assertEqual(utils.compare(123, "123", "eq"), True)
+        self.assertEqual(utils.compare(123, "123"), True)