refactor: add extract_binds and validators

ate/context.py

@@ -8,7 +8,6 @@ class Context(object):
     def __init__(self):
         self.functions = dict()
         self.variables = dict()  # Maps variable name to value
-        self.extractors = dict()
 
     def import_requires(self, modules):
         """ import required modules dynamicly
@@ -45,19 +44,10 @@ class Context(object):
             for var_name, var_value in variable_bind_map.items():
                 self.variables[var_name] = self.get_eval_value(var_value)
 
-    def bind_extractors(self, extract_binds):
-        """ Bind named extractors to value within the context.
-            value => parsed from requests.Response object
-            key => extractor name, can be used as variable in next testcases
-        @param (dict) extract_binds
-            {
-                "resp_status_code": "status_code",
-                "resp_headers": "headers",
-                "resp_headers_content_type": "headers.content-type",
-                "resp_content": "content"
-            }
+    def update_variables(self, variables_mapping):
+        """ update context variables binds with new variables mapping
         """
-        self.extractors.update(extract_binds)
+        self.variables.update(variables_mapping)
 
     def get_eval_value(self, data):
         """ evaluate data recursively, each variable in data will be evaluated.

ate/main.py

@@ -26,7 +26,7 @@ def create_suite(testset):
 
     test_runner = runner.TestRunner()
     config_dict = testset.get("config", {})
-    test_runner.pre_config(config_dict)
+    test_runner.update_context(config_dict)
     testcases = testset.get("testcases", [])
 
     for testcase in testcases:

ate/response.py

@@ -1,16 +1,18 @@
 from ate import utils, exception
 
+try:
+    basestring  # Python 2.x
+except NameError:
+    basestring = str    # Python 3.x
 
 class ResponseObject(object):
 
-    def __init__(self, resp_obj, context=None):
+    def __init__(self, resp_obj):
         """ initialize with a requests.Response object
         @param (requests.Response instance) resp_obj
-        @param (ate.context.Context instance) context
         """
         self.resp_obj = resp_obj
-        if context:
-            self.extract_response(context)
+        self.success = True
 
     def parsed_body(self):
         try:
@@ -25,49 +27,9 @@ class ResponseObject(object):
             'body': self.parsed_body()
         }
 
-    def diff_response(self, expected_resp_json):
-        diff_content = {}
-        resp_info = self.parsed_dict()
-
-        expected_status_code = expected_resp_json.get('status_code', 200)
-        if resp_info['status_code'] != int(expected_status_code):
-            diff_content['status_code'] = {
-                'value': resp_info['status_code'],
-                'expected': expected_status_code
-            }
-
-        expected_headers = expected_resp_json.get('headers', {})
-        headers_diff = utils.diff_json(resp_info['headers'], expected_headers)
-        if headers_diff:
-            diff_content['headers'] = headers_diff
-
-        expected_body = expected_resp_json.get('body', None)
-
-        if expected_body is None:
-            body_diff = {}
-        elif type(expected_body) != type(resp_info['body']):
-            body_diff = {
-                'value': resp_info['body'],
-                'expected': expected_body
-            }
-        elif isinstance(expected_body, str):
-            if expected_body != resp_info['body']:
-                body_diff = {
-                    'value': resp_info['body'],
-                    'expected': expected_body
-                }
-        elif isinstance(expected_body, dict):
-            body_diff = utils.diff_json(resp_info['body'], expected_body)
-
-        if body_diff:
-            diff_content['body'] = body_diff
-
-        return diff_content
-
-    def extract_response(self, context, delimiter='.'):
-        """ extract content from requests.Response, and bind extracted value to context.extractors
-        @param (ate.context.Context instance) context
-            context.extractors:
+    def extract_response(self, extract_binds, delimiter='.'):
+        """ extract content from requests.Response
+        @param (dict) extract_binds
             {
                 "resp_status_code": "status_code",
                 "resp_headers_content_type": "headers.content-type",
@@ -75,34 +37,73 @@ class ResponseObject(object):
                 "resp_content_person_first_name": "content.person.name.first_name"
             }
         """
-        for key, value in context.extractors.items():
+        extract_binds_dict = {}
+
+        for key, value in extract_binds.items():
+            if not isinstance(value, basestring):
+                raise exception.ParamsError("invalid extract_binds!")
+
             try:
-                if isinstance(value, str):
-                    value += "."
-                    # string.split(sep=None, maxsplit=-1) -> list of strings
-                    # e.g. "content.person.name" => ["content", "person.name"]
-                    top_query, sub_query = value.split(delimiter, 1)
-
-                    if top_query in ["body", "content", "text"]:
-                        json_content = self.parsed_body()
-                    else:
-                        json_content = getattr(self.resp_obj, top_query)
-
-                    if sub_query:
-                        # e.g. key: resp_headers_content_type, sub_query = "content-type"
-                        answer = utils.query_json(json_content, sub_query)
-                        context.extractors[key] = answer
-                    else:
-                        # e.g. key: resp_status_code, resp_content
-                        context.extractors[key] = json_content
+                value += "."
+                # string.split(sep=None, maxsplit=-1) -> list of strings
+                # e.g. "content.person.name" => ["content", "person.name"]
+                top_query, sub_query = value.split(delimiter, 1)
 
+                if top_query in ["body", "content", "text"]:
+                    json_content = self.parsed_body()
                 else:
-                    raise NotImplementedError("TODO: support template.")
+                    json_content = getattr(self.resp_obj, top_query)
+
+                if sub_query:
+                    # e.g. key: resp_headers_content_type, sub_query = "content-type"
+                    answer = utils.query_json(json_content, sub_query)
+                    extract_binds_dict[key] = answer
+                else:
+                    # e.g. key: resp_status_code, resp_content
+                    extract_binds_dict[key] = json_content
 
             except AttributeError:
                 raise exception.ParamsError("invalid extract_binds!")
 
-    def validate(self, expected_resp_json):
-        diff_content = self.diff_response(expected_resp_json)
-        success = False if diff_content else True
-        return success, diff_content
+        return extract_binds_dict
+
+    def validate(self, validators, variables_mapping):
+        """ Bind named validators to value within the context.
+        @param (dict) validators
+            {
+                "resp_status_code": {"comparator": "eq", "expected": 201},
+                "resp_body_success": {"comparator": "eq", "expected": True}
+            }
+        @param (dict) variables_mapping
+            {
+                "resp_status_code": 200,
+                "resp_body_success": True
+            }
+        @return (dict) content differences
+            {
+                "resp_status_code": {
+                    "comparator": "eq", "expected": 201, "value": 200
+                }
+            }
+        """
+        diff_content_dict = {}
+
+        for validator_key, validator_dict in validators.items():
+
+            try:
+                value = variables_mapping[validator_key]
+                validator_dict["value"] = value
+            except KeyError:
+                raise exception.ParamsError("invalid validator %s" % validator_key)
+
+            difference_exist = utils.compare(
+                value,
+                validator_dict["expected"],
+                validator_dict["comparator"]
+            )
+
+            if difference_exist:
+                diff_content_dict[validator_key] = validator_dict
+
+        self.success = False if diff_content_dict else True
+        return diff_content_dict

ate/runner.py

@@ -2,7 +2,7 @@ import requests
 
 from ate import exception, response
 from ate.context import Context
-from ate.testcase import TestcaseParser
+from ate.testcase import parse_template
 
 
 class TestRunner(object):
@@ -10,10 +10,9 @@ class TestRunner(object):
     def __init__(self):
         self.client = requests.Session()
         self.context = Context()
-        self.testcase_parser = TestcaseParser()
 
-    def pre_config(self, config_dict):
-        """ create/update variables binds
+    def update_context(self, config_dict):
+        """ create/update context variables binds
         @param config_dict
             {
                 "name": "description content",
@@ -41,44 +40,6 @@ class TestRunner(object):
         variable_binds = config_dict.get('variable_binds', [])
         self.context.bind_variables(variable_binds)
 
-        extract_binds = config_dict.get('extract_binds', {})
-        self.context.bind_extractors(extract_binds)
-
-        self.testcase_parser.update_variables_binds(self.context.variables)
-
-    def parse_testcase(self, testcase):
-        """ parse testcase with variables binds if it is a template.
-        @param (dict) testcase
-            {
-                "name": "testcase description",
-                "requires": [],  # optional, override
-                "function_binds": {}, # optional, override
-                "variable_binds": {}, # optional, override
-                "request": {},
-                "response": {}
-            }
-        @return (dict) parsed testcase with bind values
-            {
-                "request": {
-                    "url": "http://127.0.0.1:5000/api/users/1000",
-                    "method": "POST",
-                    "headers": {
-                        "Content-Type": "application/json",
-                        "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
-                        "random": "A2dEx"
-                    },
-                    "body": '{"name": "user", "password": "123456"}'
-                },
-                "response": {
-                    "status_code": 201
-                }
-            }
-        """
-        self.pre_config(testcase)
-
-        parsed_testcase = self.testcase_parser.parse(testcase)
-        return parsed_testcase
-
     def run_test(self, testcase):
         """ run single testcase.
         @param (dict) testcase
@@ -87,26 +48,42 @@ class TestRunner(object):
                 "requires": [],  # optional, override
                 "function_binds": {}, # optional, override
                 "variable_binds": {}, # optional, override
-                "request": {},
-                "response": {}
+                "request": {
+                    "url": "http://127.0.0.1:5000/api/users/1000",
+                    "method": "POST",
+                    "headers": {
+                        "Content-Type": "application/json",
+                        "authorization": "${authorization}",
+                        "random": "${random}"
+                    },
+                    "body": '{"name": "user", "password": "123456"}'
+                },
+                "extract_binds": {},
+                "validators": {}
             }
         @return (tuple) test result of single testcase
             (success, diff_content)
         """
-        testcase = self.parse_testcase(testcase)
-
-        req_kwargs = testcase['request']
+        self.update_context(testcase)
+        parsed_request = parse_template(testcase["request"], self.context.variables)
 
         try:
-            url = req_kwargs.pop('url')
-            method = req_kwargs.pop('method')
+            url = parsed_request.pop('url')
+            method = parsed_request.pop('method')
         except KeyError:
             raise exception.ParamsError("URL or METHOD missed!")
 
-        resp = self.client.request(url=url, method=method, **req_kwargs)
+        resp = self.client.request(url=url, method=method, **parsed_request)
+        resp_obj = response.ResponseObject(resp)
 
-        resp_obj = response.ResponseObject(resp, self.context)
-        return resp_obj.validate(testcase['response'])
+        extract_binds = testcase.get("extract_binds", {})
+        extract_binds_dict = resp_obj.extract_response(extract_binds)
+        self.context.update_variables(extract_binds_dict)
+
+        validators = testcase.get("validators", {})
+        diff_content_dict = resp_obj.validate(validators, self.context.variables)
+
+        return resp_obj.success, diff_content_dict
 
     def run_testset(self, testset):
         """ run single testset, including one or several testcases.
@@ -124,7 +101,8 @@ class TestRunner(object):
                         "name": "testcase description",
                         "variable_binds": {}, # override
                         "request": {},
-                        "response": {}
+                        "extract_binds": {},
+                        "validators": {}
                     },
                     testcase12
                 ]
@@ -138,7 +116,7 @@ class TestRunner(object):
         results = []
 
         config_dict = testset.get("config", {})
-        self.pre_config(config_dict)
+        self.update_context(config_dict)
         testcases = testset.get("testcases", [])
         for testcase in testcases:
             result = self.run_test(testcase)

ate/testcase.py

@@ -1,69 +1,56 @@
 from ate import utils
 
-class TestcaseParser(object):
 
-    def __init__(self, variables_binds={}):
-        self.variables_binds = variables_binds
+def parse_template(testcase_template, variables_binds):
+    """ parse testcase_template, replace all variables with bind value.
+    variables marker: ${variable}.
+    @param (dict) testcase_template
+        {
+            "url": "http://127.0.0.1:5000/api/users/${uid}",
+            "method": "POST",
+            "headers": {
+                "Content-Type": "application/json",
+                "authorization": "${authorization}",
+                "random": "${random}"
+            },
+            "body": "${data}"
+        }
+    @param (dict) variables binds mapping
+        {
+            "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
+            "random": "A2dEx",
+            "data": '{"name": "user", "password": "123456"}'
+        }
+    @return (dict) parsed testcase with bind variable values
+        {
+            "url": "http://127.0.0.1:5000/api/users/1000",
+            "method": "POST",
+            "headers": {
+                "Content-Type": "application/json",
+                "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
+                "random": "A2dEx"
+            },
+            "body": '{"name": "user", "password": "123456"}'
+        }
+    """
 
-    def update_variables_binds(self, variables_mapping):
-        """ update variables binds with new mapping.
-        """
-        if variables_mapping:
-            self.variables_binds.update(variables_mapping)
-
-    def parse(self, testcase_template):
-        """ parse testcase_template, replace all variables with bind value.
-        variables marker: ${variable}.
-        @param (dict) testcase_template
-            {
-                "request": {
-                    "url": "http://127.0.0.1:5000/api/users/${uid}",
-                    "method": "POST",
-                    "headers": {
-                        "Content-Type": "application/json",
-                        "authorization": "${authorization}",
-                        "random": "${random}"
-                    },
-                    "body": "${data}"
-                },
-                "response": {
-                    "status_code": "${expected_status}"
-                }
-            }
-        @return (dict) parsed testcase with bind values
-            {
-                "request": {
-                    "url": "http://127.0.0.1:5000/api/users/1000",
-                    "method": "POST",
-                    "headers": {
-                        "Content-Type": "application/json",
-                        "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
-                        "random": "A2dEx"
-                    },
-                    "body": '{"name": "user", "password": "123456"}'
-                },
-                "response": {
-                    "status_code": 201
-                }
-            }
-        """
-        return self.substitute(testcase_template)
-
-    def substitute(self, content):
+    def substitute(content):
         """ substitute content recursively, each variable will be replaced with bind value.
             variables marker: ${variable}.
         """
         if isinstance(content, str):
-            return utils.parse_content_with_variables(content, self.variables_binds)
+            return utils.parse_content_with_variables(content, variables_binds)
 
         if isinstance(content, list):
-            return [self.substitute(item) for item in content]
+            return [substitute(item) for item in content]
 
         if isinstance(content, dict):
             parsed_content = {}
             for key, value in content.items():
-                parsed_content[key] = self.substitute(value)
+                parsed_content[key] = substitute(value)
 
             return parsed_content
 
         return content
+
+    return substitute(testcase_template)

ate/utils.py

@@ -62,19 +62,6 @@ def load_testcases(testcase_file_path):
         # '' or other suffix
         raise ParamsError("Bad testcase file name!")
 
-def diff_json(current_json, expected_json):
-    json_diff = {}
-
-    for key, expected_value in expected_json.items():
-        value = current_json.get(key, None)
-        if str(value) != str(expected_value):
-            json_diff[key] = {
-                'value': value,
-                'expected': expected_value
-            }
-
-    return json_diff
-
 def load_foler_files(folder_path):
     """ load folder path, return all files in list format.
     """
@@ -192,3 +179,25 @@ def query_json(json_content, query, delimiter='.'):
         raise ParamsError("invalid query string in extract_binds!")
 
     return json_content
+
+def diff_json(current_json, expected_json):
+    json_diff = {}
+
+    for key, expected_value in expected_json.items():
+        value = current_json.get(key, None)
+        if str(value) != str(expected_value):
+            json_diff[key] = {
+                'value': value,
+                'expected': expected_value
+            }
+
+    return json_diff
+
+def compare(value, expected, comparator="eq"):
+    try:
+        if comparator in ["eq", "=="]:
+            assert value == expected
+
+        return False
+    except AssertionError:
+        return True