refactor: split validate from context, move to validator

httprunner/api.py

@@ -277,9 +277,9 @@ class HttpRunner(object):
 
         """
         logger.log_info("HttpRunner version: {}".format(__version__))
-        if validator.is_testcase_path(path_or_tests):
+        if loader.is_testcase_path(path_or_tests):
             return self.run_path(path_or_tests, dot_env_path, mapping)
-        elif validator.is_testcases(path_or_tests):
+        elif loader.is_testcases(path_or_tests):
             return self.run_tests(path_or_tests)
         else:
             raise exceptions.ParamsError("Invalid testcase path or testcases: {}".format(path_or_tests))

httprunner/context.py

@@ -1,4 +1,4 @@
-from httprunner import exceptions, logger, parser, utils
+from httprunner import parser, utils
 
 
 class SessionContext(object):
@@ -19,7 +19,6 @@ class SessionContext(object):
         self.session_variables_mapping = parser.parse_variables_mapping(variables_mapping)
         self.test_variables_mapping = {}
         self.init_test_variables()
-        self.validation_results = {}
 
     def init_test_variables(self, variables_mapping=None):
         """ init test variables, called when each test(api) starts.
@@ -63,175 +62,3 @@ class SessionContext(object):
             content may be in any data structure, include dict, list, tuple, number, string, etc.
         """
         return parser.parse_lazy_data(content, self.test_variables_mapping)
-
-    def __eval_validator_check(self, check_item, resp_obj):
-        """ evaluate check item in validator.
-
-        Args:
-            check_item: check_item should only be the following 5 formats:
-                1, variable reference, e.g. $token
-                2, function reference, e.g. ${is_status_code_200($status_code)}
-                3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
-                4, string joined by delimiter. e.g. "status_code", "headers.content-type"
-                5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
-
-            resp_obj: response object
-
-        """
-        if isinstance(check_item, (dict, list)) \
-                or isinstance(check_item, parser.LazyString):
-            # format 1/2/3
-            check_value = self.eval_content(check_item)
-        else:
-            # format 4/5
-            check_value = resp_obj.extract_field(check_item)
-
-        return check_value
-
-    def __eval_validator_expect(self, expect_item):
-        """ evaluate expect item in validator.
-
-        Args:
-            expect_item: expect_item should only be in 2 types:
-                1, variable reference, e.g. $expect_status_code
-                2, actual value, e.g. 200
-
-        """
-        expect_value = self.eval_content(expect_item)
-        return expect_value
-
-    def validate(self, validators, resp_obj):
-        """ make validation with comparators
-        """
-        self.validation_results = {}
-        if not validators:
-            return
-
-        logger.log_debug("start to validate.")
-
-        validate_pass = True
-        failures = []
-
-        for validator in validators:
-
-            if isinstance(validator, dict) and validator.get("type") == "python_script":
-                validator_dict, ex = self.validate_script(validator["script"], resp_obj)
-                if ex:
-                    validate_pass = False
-                    failures.append(ex)
-
-                self.validation_results["validate_script"] = validator_dict
-                continue
-
-            if "validate_extractor" not in self.validation_results:
-                self.validation_results["validate_extractor"] = []
-
-            # validator should be LazyFunction object
-            if not isinstance(validator, parser.LazyFunction):
-                raise exceptions.ValidationFailure(
-                    "validator should be parsed first: {}".format(validators))
-
-            # evaluate validator args with context variable mapping.
-            validator_args = validator.get_args()
-            check_item, expect_item = validator_args
-            check_value = self.__eval_validator_check(
-                check_item,
-                resp_obj
-            )
-            expect_value = self.__eval_validator_expect(expect_item)
-            validator.update_args([check_value, expect_value])
-
-            comparator = validator.func_name
-            validator_dict = {
-                "comparator": comparator,
-                "check": check_item,
-                "check_value": check_value,
-                "expect": expect_item,
-                "expect_value": expect_value
-            }
-            validate_msg = "\nvalidate: {} {} {}({})".format(
-                check_item,
-                comparator,
-                expect_value,
-                type(expect_value).__name__
-            )
-
-            try:
-                validator.to_value(self.test_variables_mapping)
-                validator_dict["check_result"] = "pass"
-                validate_msg += "\t==> pass"
-                logger.log_debug(validate_msg)
-            except (AssertionError, TypeError):
-                validate_pass = False
-                validator_dict["check_result"] = "fail"
-                validate_msg += "\t==> fail"
-                validate_msg += "\n{}({}) {} {}({})".format(
-                    check_value,
-                    type(check_value).__name__,
-                    comparator,
-                    expect_value,
-                    type(expect_value).__name__
-                )
-                logger.log_error(validate_msg)
-                failures.append(validate_msg)
-
-            self.validation_results["validate_extractor"].append(validator_dict)
-
-            # restore validator args, in case of running multiple times
-            validator.update_args(validator_args)
-
-        if not validate_pass:
-            failures_string = "\n".join([failure for failure in failures])
-            raise exceptions.ValidationFailure(failures_string)
-
-    def validate_script(self, script, resp_obj):
-        """ make validation with python script
-        """
-        validator_dict = {
-            "validate_script": "<br/>".join(script),
-            "check_result": "fail",
-            "exception": ""
-        }
-
-        script = "\n    ".join(script)
-        code = """
-# encoding: utf-8
-
-try:
-    {}
-except Exception as ex:
-    import traceback
-    import sys
-    _type, _value, _tb = sys.exc_info()
-    # filename, lineno, name, line
-    _, _lineno, _, line_content = traceback.extract_tb(_tb, 1)[0]
-
-    line_no = _lineno - 4
-
-    c_exception = _type.__name__ + "\\n"
-    c_exception += "\\tError line number: " + str(line_no) + "\\n"
-    c_exception += "\\tError line content: " + str(line_content) + "\\n"
-
-    if _value.args:
-        c_exception += "\\tError description: " + str(_value)
-    else:
-        c_exception += "\\tError description: " + _type.__name__
-
-    raise _type(c_exception)
-""".format(script)
-        variables = {
-            "status_code": resp_obj.status_code,
-            "response_json": resp_obj.json,
-            "response": resp_obj
-        }
-        variables.update(self.test_variables_mapping)
-
-        try:
-            code = compile(code, '<string>', 'exec')
-            exec(code, variables)
-            validator_dict["check_result"] = "pass"
-            return validator_dict, ""
-        except Exception as ex:
-            validator_dict["check_result"] = "fail"
-            validator_dict["exception"] = "<br/>".join(str(ex).splitlines())
-            return validator_dict, str(ex)

httprunner/runner.py

@@ -5,6 +5,7 @@ from unittest.case import SkipTest
 from httprunner import exceptions, logger, response, utils
 from httprunner.client import HttpSession
 from httprunner.context import SessionContext
+from httprunner.validator import Validator
 
 
 class Runner(object):
@@ -87,16 +88,6 @@ class Runner(object):
 
         self.http_client_session.init_meta_data()
 
-    def __get_test_data(self):
-        """ get request/response data and validate results
-        """
-        if not isinstance(self.http_client_session, HttpSession):
-            return
-
-        meta_data = self.http_client_session.meta_data
-        meta_data["validators"] = self.session_context.validation_results
-        return meta_data
-
     def _handle_skip_feature(self, test_dict):
         """ handle skip feature for test
             - skip: skip current test unconditionally
@@ -267,7 +258,6 @@ class Runner(object):
         self.session_context.update_session_variables(extracted_variables_mapping)
 
         # validate
-        # TODO: split validate from context
         validators = test_dict.get("validate") or test_dict.get("validators") or []
         validate_script = test_dict.get("validate_script", [])
         if validate_script:
@@ -276,8 +266,9 @@ class Runner(object):
                 "script": validate_script
             })
 
+        validator = Validator(self.session_context, resp_obj)
         try:
-            self.session_context.validate(validators, resp_obj)
+            validator.validate(validators)
         except (exceptions.ParamsError,
                 exceptions.ValidationFailure, exceptions.ExtractFailure):
             err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32)
@@ -302,6 +293,11 @@ class Runner(object):
 
             raise
 
+        finally:
+            # get request/response data and validate results
+            self.meta_datas = getattr(self.http_client_session, "meta_data", {})
+            self.meta_datas["validators"] = validator.validation_results
+
     def _run_testcase(self, testcase_dict):
         """ run single testcase.
         """
@@ -385,8 +381,6 @@ class Runner(object):
                 self.exception_request_type = test_dict["request"]["method"]
                 self.exception_name = test_dict.get("name")
                 raise
-            finally:
-                self.meta_datas = self.__get_test_data()
 
     def export_variables(self, output_variables_list):
         """ export current testcase variables
@@ -397,8 +391,8 @@ class Runner(object):
         for variable in output_variables_list:
             if variable not in variables_mapping:
                 logger.log_warning(
-                    "variable '{}' can not be found in variables mapping, failed to export!"\
+                    "variable '{}' can not be found in variables mapping, "
-                        .format(variable)
+                    "failed to export!".format(variable)
                 )
                 continue
 

httprunner/validator.py

@@ -2,136 +2,9 @@
 import collections
 import io
 import json
-import os
 import types
 
-from httprunner import exceptions, logger
+from httprunner import exceptions, logger, parser
-
-""" validate data format
-TODO: refactor with JSON schema validate
-"""
-
-
-def is_testcase(data_structure):
-    """ check if data_structure is a testcase.
-
-    Args:
-        data_structure (dict): testcase should always be in the following data structure:
-
-            {
-                "config": {
-                    "name": "desc1",
-                    "variables": [],    # optional
-                    "request": {}       # optional
-                },
-                "teststeps": [
-                    test_dict1,
-                    {   # test_dict2
-                        'name': 'test step desc2',
-                        'variables': [],    # optional
-                        'extract': [],      # optional
-                        'validate': [],
-                        'request': {},
-                        'function_meta': {}
-                    }
-                ]
-            }
-
-    Returns:
-        bool: True if data_structure is valid testcase, otherwise False.
-
-    """
-    # TODO: replace with JSON schema validation
-    if not isinstance(data_structure, dict):
-        return False
-
-    if "teststeps" not in data_structure:
-        return False
-
-    if not isinstance(data_structure["teststeps"], list):
-        return False
-
-    return True
-
-
-def is_testcases(data_structure):
-    """ check if data_structure is testcase or testcases list.
-
-    Args:
-        data_structure (dict): testcase(s) should always be in the following data structure:
-            {
-                "project_mapping": {
-                    "PWD": "XXXXX",
-                    "functions": {},
-                    "env": {}
-                },
-                "testcases": [
-                    {   # testcase data structure
-                        "config": {
-                            "name": "desc1",
-                            "path": "testcase1_path",
-                            "variables": [],                    # optional
-                        },
-                        "teststeps": [
-                            # test data structure
-                            {
-                                'name': 'test step desc1',
-                                'variables': [],    # optional
-                                'extract': [],      # optional
-                                'validate': [],
-                                'request': {}
-                            },
-                            test_dict_2   # another test dict
-                        ]
-                    },
-                    testcase_dict_2     # another testcase dict
-                ]
-            }
-
-    Returns:
-        bool: True if data_structure is valid testcase(s), otherwise False.
-
-    """
-    if not isinstance(data_structure, dict):
-        return False
-
-    if "testcases" not in data_structure:
-        return False
-
-    testcases = data_structure["testcases"]
-    if not isinstance(testcases, list):
-        return False
-
-    for item in testcases:
-        if not is_testcase(item):
-            return False
-
-    return True
-
-
-def is_testcase_path(path):
-    """ check if path is testcase path or path list.
-
-    Args:
-        path (str/list): file path or file path list.
-
-    Returns:
-        bool: True if path is valid file path or path list, otherwise False.
-
-    """
-    if not isinstance(path, (str, list)):
-        return False
-
-    if isinstance(path, list):
-        for p in path:
-            if not is_testcase_path(p):
-                return False
-
-    if isinstance(path, str):
-        if not os.path.exists(path):
-            return False
-
-    return True
 
 
 ###############################################################################
@@ -345,3 +218,191 @@ def validate_json_file(file_list):
                 raise SystemExit(e)
 
         print("OK")
+
+
+class Validator(object):
+    """Validate tests
+
+    Attributes:
+        validation_results (dict): store validation results,
+            including validate_extractor and validate_script.
+
+    """
+
+    def __init__(self, session_context, resp_obj):
+        """ initialize a Validator for each teststep (API request)
+
+        Args:
+            session_context: HttpRunner session context
+            resp_obj: ResponseObject instance
+        """
+        self.session_context = session_context
+        self.resp_obj = resp_obj
+        self.validation_results = {}
+
+    def __eval_validator_check(self, check_item):
+        """ evaluate check item in validator.
+
+        Args:
+            check_item: check_item should only be the following 5 formats:
+                1, variable reference, e.g. $token
+                2, function reference, e.g. ${is_status_code_200($status_code)}
+                3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
+                4, string joined by delimiter. e.g. "status_code", "headers.content-type"
+                5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
+
+        """
+        if isinstance(check_item, (dict, list)) \
+                or isinstance(check_item, parser.LazyString):
+            # format 1/2/3
+            check_value = self.session_context.eval_content(check_item)
+        else:
+            # format 4/5
+            check_value = self.resp_obj.extract_field(check_item)
+
+        return check_value
+
+    def __eval_validator_expect(self, expect_item):
+        """ evaluate expect item in validator.
+
+        Args:
+            expect_item: expect_item should only be in 2 types:
+                1, variable reference, e.g. $expect_status_code
+                2, actual value, e.g. 200
+
+        """
+        expect_value = self.session_context.eval_content(expect_item)
+        return expect_value
+
+    def validate_script(self, script):
+        """ make validation with python script
+        """
+        validator_dict = {
+            "validate_script": "<br/>".join(script),
+            "check_result": "fail",
+            "exception": ""
+        }
+
+        script = "\n    ".join(script)
+        code = """
+# encoding: utf-8
+
+try:
+    {}
+except Exception as ex:
+    import traceback
+    import sys
+    _type, _value, _tb = sys.exc_info()
+    # filename, lineno, name, line
+    _, _lineno, _, line_content = traceback.extract_tb(_tb, 1)[0]
+
+    line_no = _lineno - 4
+
+    c_exception = _type.__name__ + "\\n"
+    c_exception += "\\tError line number: " + str(line_no) + "\\n"
+    c_exception += "\\tError line content: " + str(line_content) + "\\n"
+
+    if _value.args:
+        c_exception += "\\tError description: " + str(_value)
+    else:
+        c_exception += "\\tError description: " + _type.__name__
+
+    raise _type(c_exception)
+""".format(script)
+        variables = {
+            "status_code": self.resp_obj.status_code,
+            "response_json": self.resp_obj.json,
+            "response": self.resp_obj
+        }
+        variables.update(self.session_context.test_variables_mapping)
+
+        try:
+            code = compile(code, '<string>', 'exec')
+            exec(code, variables)
+            validator_dict["check_result"] = "pass"
+            return validator_dict, ""
+        except Exception as ex:
+            validator_dict["check_result"] = "fail"
+            validator_dict["exception"] = "<br/>".join(str(ex).splitlines())
+            return validator_dict, str(ex)
+
+    def validate(self, validators):
+        """ make validation with comparators
+        """
+        self.validation_results = {}
+        if not validators:
+            return
+
+        logger.log_debug("start to validate.")
+
+        validate_pass = True
+        failures = []
+
+        for validator in validators:
+
+            if isinstance(validator, dict) and validator.get("type") == "python_script":
+                validator_dict, ex = self.validate_script(validator["script"])
+                if ex:
+                    validate_pass = False
+                    failures.append(ex)
+
+                self.validation_results["validate_script"] = validator_dict
+                continue
+
+            if "validate_extractor" not in self.validation_results:
+                self.validation_results["validate_extractor"] = []
+
+            # validator should be LazyFunction object
+            if not isinstance(validator, parser.LazyFunction):
+                raise exceptions.ValidationFailure(
+                    "validator should be parsed first: {}".format(validators))
+
+            # evaluate validator args with context variable mapping.
+            validator_args = validator.get_args()
+            check_item, expect_item = validator_args
+            check_value = self.__eval_validator_check(check_item)
+            expect_value = self.__eval_validator_expect(expect_item)
+            validator.update_args([check_value, expect_value])
+
+            comparator = validator.func_name
+            validator_dict = {
+                "comparator": comparator,
+                "check": check_item,
+                "check_value": check_value,
+                "expect": expect_item,
+                "expect_value": expect_value
+            }
+            validate_msg = "\nvalidate: {} {} {}({})".format(
+                check_item,
+                comparator,
+                expect_value,
+                type(expect_value).__name__
+            )
+
+            try:
+                validator.to_value(self.session_context.test_variables_mapping)
+                validator_dict["check_result"] = "pass"
+                validate_msg += "\t==> pass"
+                logger.log_debug(validate_msg)
+            except (AssertionError, TypeError):
+                validate_pass = False
+                validator_dict["check_result"] = "fail"
+                validate_msg += "\t==> fail"
+                validate_msg += "\n{}({}) {} {}({})".format(
+                    check_value,
+                    type(check_value).__name__,
+                    comparator,
+                    expect_value,
+                    type(expect_value).__name__
+                )
+                logger.log_error(validate_msg)
+                failures.append(validate_msg)
+
+            self.validation_results["validate_extractor"].append(validator_dict)
+
+            # restore validator args, in case of running multiple times
+            validator.update_args(validator_args)
+
+        if not validate_pass:
+            failures_string = "\n".join([failure for failure in failures])
+            raise exceptions.ValidationFailure(failures_string)