Merge pull request #1 from httprunner/dev

Dev

docs/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Release History
 
+## 2.4.0 (2019-12-04)
+
+**Added**
+
+- feat: validate with python script, ref #773
+
+## 2.3.3 (2019-12-04)
+
+**Fixed**
+
+- fix #768: dump json file path error when folder name contains dot, such as `a.b.c`
+
+**Changed**
+
+- change: rename builtin function, sleep_N_secs => sleep
+
 ## 2.3.2 (2019-11-01)
 
 **Added**

httprunner/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.3.2"
+__version__ = "2.4.0"
 __description__ = "One-stop solution for HTTP(S) testing."
 
 __all__ = ["__version__", "__description__"]

httprunner/api.py

@@ -258,7 +258,6 @@ class HttpRunner(object):
         # load tests
         self.exception_stage = "load tests"
         tests_mapping = loader.load_tests(path, dot_env_path)
-        tests_mapping["project_mapping"]["test_path"] = path
 
         if mapping:
             tests_mapping["project_mapping"]["variables"] = mapping
@@ -278,9 +277,9 @@ class HttpRunner(object):
 
         """
         logger.log_info("HttpRunner version: {}".format(__version__))
-        if validator.is_testcase_path(path_or_tests):
+        if loader.is_testcase_path(path_or_tests):
             return self.run_path(path_or_tests, dot_env_path, mapping)
-        elif validator.is_testcases(path_or_tests):
+        elif loader.is_testcases(path_or_tests):
             return self.run_tests(path_or_tests)
         else:
             raise exceptions.ParamsError("Invalid testcase path or testcases: {}".format(path_or_tests))

httprunner/cli.py

@@ -5,11 +5,11 @@ import sys
 from httprunner import __description__, __version__
 from httprunner.api import HttpRunner
 from httprunner.compat import is_py2
+from httprunner.loader import validate_json_file
 from httprunner.logger import color_print
 from httprunner.report import gen_html_report
 from httprunner.utils import (create_scaffold, get_python2_retire_msg,
                               prettify_json_file)
-from httprunner.validator import validate_json_file
 
 
 def main():

httprunner/client.py

@@ -8,7 +8,7 @@ from requests import Request, Response
 from requests.exceptions import (InvalidSchema, InvalidURL, MissingSchema,
                                  RequestException)
 
-from httprunner import logger
+from httprunner import logger, response
 from httprunner.utils import lower_dict_keys, omit_long_data
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -115,7 +115,10 @@ class HttpSession(requests.Session):
         else:
             try:
                 # try to record json data
-                req_resp_dict["response"]["json"] = resp_obj.json()
+                if isinstance(resp_obj, response.ResponseObject):
+                    req_resp_dict["response"]["json"] = resp_obj.json
+                else:
+                    req_resp_dict["response"]["json"] = resp_obj.json()
             except ValueError:
                 # only record at most 512 text charactors
                 resp_text = resp_obj.text
@@ -126,6 +129,13 @@ class HttpSession(requests.Session):
 
         return req_resp_dict
 
+    def update_last_req_resp_record(self, resp_obj):
+        """
+        update request and response info from Response() object.
+        """
+        self.meta_data["data"].pop()
+        self.meta_data["data"].append(self.get_req_resp_record(resp_obj))
+
     def request(self, method, url, name=None, **kwargs):
         """
         Constructs and sends a :py:class:`requests.Request`.

httprunner/context.py

@@ -1,4 +1,4 @@
-from httprunner import exceptions, logger, parser, utils
+from httprunner import parser, utils
 
 
 class SessionContext(object):
@@ -13,11 +13,12 @@ class SessionContext(object):
         >>> context.update_session_variables(variables)
 
     """
+
     def __init__(self, variables=None):
         variables_mapping = utils.ensure_mapping_format(variables or {})
         self.session_variables_mapping = parser.parse_variables_mapping(variables_mapping)
+        self.test_variables_mapping = {}
         self.init_test_variables()
-        self.validation_results = []
 
     def init_test_variables(self, variables_mapping=None):
         """ init test variables, called when each test(api) starts.
@@ -61,110 +62,3 @@ class SessionContext(object):
             content may be in any data structure, include dict, list, tuple, number, string, etc.
         """
         return parser.parse_lazy_data(content, self.test_variables_mapping)
-
-    def __eval_validator_check(self, check_item, resp_obj):
-        """ evaluate check item in validator.
-
-        Args:
-            check_item: check_item should only be the following 5 formats:
-                1, variable reference, e.g. $token
-                2, function reference, e.g. ${is_status_code_200($status_code)}
-                3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
-                4, string joined by delimiter. e.g. "status_code", "headers.content-type"
-                5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
-
-            resp_obj: response object
-
-        """
-        if isinstance(check_item, (dict, list)) \
-            or isinstance(check_item, parser.LazyString):
-            # format 1/2/3
-            check_value = self.eval_content(check_item)
-        else:
-            # format 4/5
-            check_value = resp_obj.extract_field(check_item)
-
-        return check_value
-
-    def __eval_validator_expect(self, expect_item):
-        """ evaluate expect item in validator.
-
-        Args:
-            expect_item: expect_item should only be in 2 types:
-                1, variable reference, e.g. $expect_status_code
-                2, actual value, e.g. 200
-
-        """
-        expect_value = self.eval_content(expect_item)
-        return expect_value
-
-    def validate(self, validators, resp_obj):
-        """ make validation with comparators
-        """
-        self.validation_results = []
-        if not validators:
-            return
-
-        logger.log_debug("start to validate.")
-
-        validate_pass = True
-        failures = []
-
-        for validator in validators:
-            # validator should be LazyFunction object
-            if not isinstance(validator, parser.LazyFunction):
-                raise exceptions.ValidationFailure(
-                    "validator should be parsed first: {}".format(validators))
-
-            # evaluate validator args with context variable mapping.
-            validator_args = validator.get_args()
-            check_item, expect_item = validator_args
-            check_value = self.__eval_validator_check(
-                check_item,
-                resp_obj
-            )
-            expect_value = self.__eval_validator_expect(expect_item)
-            validator.update_args([check_value, expect_value])
-
-            comparator = validator.func_name
-            validator_dict = {
-                "comparator": comparator,
-                "check": check_item,
-                "check_value": check_value,
-                "expect": expect_item,
-                "expect_value": expect_value
-            }
-            validate_msg = "\nvalidate: {} {} {}({})".format(
-                check_item,
-                comparator,
-                expect_value,
-                type(expect_value).__name__
-            )
-
-            try:
-                validator.to_value(self.test_variables_mapping)
-                validator_dict["check_result"] = "pass"
-                validate_msg += "\t==> pass"
-                logger.log_debug(validate_msg)
-            except (AssertionError, TypeError):
-                validate_pass = False
-                validator_dict["check_result"] = "fail"
-                validate_msg += "\t==> fail"
-                validate_msg += "\n{}({}) {} {}({})".format(
-                    check_value,
-                    type(check_value).__name__,
-                    comparator,
-                    expect_value,
-                    type(expect_value).__name__
-                )
-                logger.log_error(validate_msg)
-                failures.append(validate_msg)
-
-            self.validation_results.append(validator_dict)
-
-            # restore validator args, in case of running multiple times
-            validator.update_args(validator_args)
-
-        if not validate_pass:
-            failures_string = "\n".join([failure for failure in failures])
-            raise exceptions.ValidationFailure(failures_string)

httprunner/loader.py

@@ -4,6 +4,7 @@ import io
 import json
 import os
 import sys
+import types
 
 import yaml
 
@@ -17,6 +18,149 @@ except AttributeError:
     pass
 
 
+# TODO: validate data format with JSON schema
+
+def is_testcase(data_structure):
+    """ check if data_structure is a testcase.
+
+    Args:
+        data_structure (dict): testcase should always be in the following data structure:
+
+            {
+                "config": {
+                    "name": "desc1",
+                    "variables": [],    # optional
+                    "request": {}       # optional
+                },
+                "teststeps": [
+                    test_dict1,
+                    {   # test_dict2
+                        'name': 'test step desc2',
+                        'variables': [],    # optional
+                        'extract': [],      # optional
+                        'validate': [],
+                        'request': {},
+                        'function_meta': {}
+                    }
+                ]
+            }
+
+    Returns:
+        bool: True if data_structure is valid testcase, otherwise False.
+
+    """
+    # TODO: replace with JSON schema validation
+    if not isinstance(data_structure, dict):
+        return False
+
+    if "teststeps" not in data_structure:
+        return False
+
+    if not isinstance(data_structure["teststeps"], list):
+        return False
+
+    return True
+
+
+def is_testcases(data_structure):
+    """ check if data_structure is testcase or testcases list.
+
+    Args:
+        data_structure (dict): testcase(s) should always be in the following data structure:
+            {
+                "project_mapping": {
+                    "PWD": "XXXXX",
+                    "functions": {},
+                    "env": {}
+                },
+                "testcases": [
+                    {   # testcase data structure
+                        "config": {
+                            "name": "desc1",
+                            "path": "testcase1_path",
+                            "variables": [],                    # optional
+                        },
+                        "teststeps": [
+                            # test data structure
+                            {
+                                'name': 'test step desc1',
+                                'variables': [],    # optional
+                                'extract': [],      # optional
+                                'validate': [],
+                                'request': {}
+                            },
+                            test_dict_2   # another test dict
+                        ]
+                    },
+                    testcase_dict_2     # another testcase dict
+                ]
+            }
+
+    Returns:
+        bool: True if data_structure is valid testcase(s), otherwise False.
+
+    """
+    if not isinstance(data_structure, dict):
+        return False
+
+    if "testcases" not in data_structure:
+        return False
+
+    testcases = data_structure["testcases"]
+    if not isinstance(testcases, list):
+        return False
+
+    for item in testcases:
+        if not is_testcase(item):
+            return False
+
+    return True
+
+
+def is_testcase_path(path):
+    """ check if path is testcase path or path list.
+
+    Args:
+        path (str/list): file path or file path list.
+
+    Returns:
+        bool: True if path is valid file path or path list, otherwise False.
+
+    """
+    if not isinstance(path, (str, list)):
+        return False
+
+    if isinstance(path, list):
+        for p in path:
+            if not is_testcase_path(p):
+                return False
+
+    if isinstance(path, str):
+        if not os.path.exists(path):
+            return False
+
+    return True
+
+
+def validate_json_file(file_list):
+    """ validate JSON testcase format
+    """
+    for json_file in set(file_list):
+        if not json_file.endswith(".json"):
+            logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file))
+            continue
+
+        logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN")
+
+        with io.open(json_file) as stream:
+            try:
+                json.load(stream)
+            except ValueError as e:
+                raise SystemExit(e)
+
+        print("OK")
+
+
 ###############################################################################
 #   file loader
 ###############################################################################
@@ -244,6 +388,31 @@ def locate_file(start_path, file_name):
 ###############################################################################
 
 
+def is_function(item):
+    """ Takes item object, returns True if it is a function.
+    """
+    return isinstance(item, types.FunctionType)
+
+
+def is_variable(tup):
+    """ Takes (name, object) tuple, returns True if it is a variable.
+    """
+    name, item = tup
+    if callable(item):
+        # function or class
+        return False
+
+    if isinstance(item, types.ModuleType):
+        # imported module
+        return False
+
+    if name.startswith("_"):
+        # private property
+        return False
+
+    return True
+
+
 def load_module_functions(module):
     """ load python module functions.
 
@@ -262,7 +431,7 @@ def load_module_functions(module):
     module_functions = {}
 
     for name, item in vars(module).items():
-        if validator.is_function(item):
+        if is_function(item):
             module_functions[name] = item
 
     return module_functions
@@ -724,7 +893,7 @@ def load_api_folder(api_folder_path):
             for api_item in api_items:
                 key, api_dict = api_item.popitem()
                 api_id = api_dict.get("id") or api_dict.get("def") \
-                    or api_dict.get("name")
+                         or api_dict.get("name")
                 if key != "api" or not api_id:
                     raise exceptions.ParamsError(
                         "Invalid API defined in {}".format(api_file_path))
@@ -779,6 +948,19 @@ def load_project_tests(test_path, dot_env_path=None):
             environments and debugtalk.py functions.
 
     """
+
+    def prepare_path(path):
+        if not os.path.exists(path):
+            err_msg = "path not exist: {}".format(path)
+            logger.log_error(err_msg)
+            raise exceptions.FileNotFound(err_msg)
+
+        if not os.path.isabs(path):
+            path = os.path.join(os.getcwd(), path)
+
+        return path
+
+    test_path = prepare_path(test_path)
     # locate debugtalk.py file
     debugtalk_path = locate_debugtalk_py(test_path)
 
@@ -810,6 +992,7 @@ def load_project_tests(test_path, dot_env_path=None):
     project_mapping["PWD"] = project_working_directory
     built_in.PWD = project_working_directory
     project_mapping["functions"] = debugtalk_functions
+    project_mapping["test_path"] = test_path
 
     # load api
     tests_def_mapping["api"] = load_api_folder(os.path.join(project_working_directory, "api"))
@@ -869,14 +1052,6 @@ def load_tests(path, dot_env_path=None):
             }
 
     """
-    if not os.path.exists(path):
-        err_msg = "path not exist: {}".format(path)
-        logger.log_error(err_msg)
-        raise exceptions.FileNotFound(err_msg)
-
-    if not os.path.isabs(path):
-        path = os.path.join(os.getcwd(), path)
-
     load_project_tests(path, dot_env_path)
     tests_mapping = {
         "project_mapping": project_mapping

httprunner/runner.py

@@ -5,6 +5,7 @@ from unittest.case import SkipTest
 from httprunner import exceptions, logger, response, utils
 from httprunner.client import HttpSession
 from httprunner.context import SessionContext
+from httprunner.validator import Validator
 
 
 class Runner(object):
@@ -62,7 +63,6 @@ class Runner(object):
         """
         self.verify = config.get("verify", True)
         self.export = config.get("export") or config.get("output", [])
-        self.validation_results = []
         config_variables = config.get("variables", {})
 
         # testcase setup hooks
@@ -86,19 +86,8 @@ class Runner(object):
         if not isinstance(self.http_client_session, HttpSession):
             return
 
-        self.validation_results = []
         self.http_client_session.init_meta_data()
 
-    def __get_test_data(self):
-        """ get request/response data and validate results
-        """
-        if not isinstance(self.http_client_session, HttpSession):
-            return
-
-        meta_data = self.http_client_session.meta_data
-        meta_data["validators"] = self.validation_results
-        return meta_data
-
     def _handle_skip_feature(self, test_dict):
         """ handle skip feature for test
             - skip: skip current test unconditionally
@@ -244,7 +233,8 @@ class Runner(object):
             raise exceptions.ParamsError(err_msg)
 
         logger.log_info("{method} {url}".format(method=method, url=parsed_url))
-        logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
+        logger.log_debug(
+            "request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
 
         # request
         resp = self.http_client_session.request(
@@ -260,6 +250,7 @@ class Runner(object):
         if teardown_hooks:
             self.session_context.update_test_variables("response", resp_obj)
             self.do_hook_actions(teardown_hooks, "teardown")
+            self.http_client_session.update_last_req_resp_record(resp_obj)
 
         # extract
         extractors = test_dict.get("extract", {})
@@ -268,9 +259,18 @@ class Runner(object):
 
         # validate
         validators = test_dict.get("validate") or test_dict.get("validators") or []
+        validate_script = test_dict.get("validate_script", [])
+        if validate_script:
+            validators.append({
+                "type": "python_script",
+                "script": validate_script
+            })
+
+        validator = Validator(self.session_context, resp_obj)
         try:
-            self.session_context.validate(validators, resp_obj)
+            validator.validate(validators)
-        except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure):
+        except (exceptions.ParamsError,
+                exceptions.ValidationFailure, exceptions.ExtractFailure):
             err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32)
 
             # log request
@@ -294,7 +294,9 @@ class Runner(object):
             raise
 
         finally:
-            self.validation_results = self.session_context.validation_results
+            # get request/response data and validate results
+            self.meta_datas = getattr(self.http_client_session, "meta_data", {})
+            self.meta_datas["validators"] = validator.validation_results
 
     def _run_testcase(self, testcase_dict):
         """ run single testcase.
@@ -379,8 +381,6 @@ class Runner(object):
                 self.exception_request_type = test_dict["request"]["method"]
                 self.exception_name = test_dict.get("name")
                 raise
-            finally:
-                self.meta_datas = self.__get_test_data()
 
     def export_variables(self, output_variables_list):
         """ export current testcase variables
@@ -391,8 +391,8 @@ class Runner(object):
         for variable in output_variables_list:
             if variable not in variables_mapping:
                 logger.log_warning(
-                    "variable '{}' can not be found in variables mapping, failed to export!"\
+                    "variable '{}' can not be found in variables mapping, "
-                        .format(variable)
+                    "failed to export!".format(variable)
                 )
                 continue
 

httprunner/static/report_template.html

@@ -279,15 +279,17 @@
               {% endfor %}
 
               <h3>Validators:</h3>
-              <div style="overflow: auto">
+                <div style="overflow: auto">
-                  <table>
+                {% set validate_extractors = meta_data.validators.validate_extractor %}
+                {% if validate_extractors %}
+                <table>
                     <tr>
                       <th>check</th>
                       <th>comparator</th>
                       <th>expect value</th>
                       <th>actual value</th>
                     </tr>
-                    {% for validator in meta_data.validators %}
+                    {% for validator in validate_extractors %}
                     <tr>
                       {% if validator.check_result == "pass" %}
                       <td class="passed">
@@ -303,7 +305,27 @@
                       <td>{{validator.check_value | e}}</td>
                     </tr>
                     {% endfor %}
-                  </table>
+                </table>
+                {% endif %}
+
+                {% set validate_script = meta_data.validators.validate_script %}
+                {% if validate_script %}
+                    <table>
+                        <tr>
+                            <th>validate script</th><th>exception</th>
+                        </tr>
+                        <tr>
+                            <td>{{validate_script.validate_script | safe}}</td>
+                            {% if validate_script.check_result == "pass" %}
+                            <td class="passed">
+                            {% elif validate_script.check_result == "fail" %}
+                            <td class="failed">
+                            {% endif %}
+                            {{validate_script.exception}}
+                            </td>
+                        </tr>
+                    </table>
+                {% endif %}
               </div>
 
               <h3>Statistics:</h3>

httprunner/utils.py

@@ -607,7 +607,7 @@ def omit_long_data(body, omit_len=512):
     return omitted_body + appendix_str
 
 
-def dump_json_file(json_data, pwd_dir_path, dump_file_name):
+def dump_json_file(json_data, json_file_abs_path):
     """ dump json data to file
     """
     class PythonObjectEncoder(json.JSONEncoder):
@@ -617,14 +617,8 @@ def dump_json_file(json_data, pwd_dir_path, dump_file_name):
             except TypeError:
                 return str(obj)
 
-    logs_dir_path = os.path.join(pwd_dir_path, "logs")
-    if not os.path.isdir(logs_dir_path):
-        os.makedirs(logs_dir_path)
-
-    dump_file_path = os.path.join(logs_dir_path, dump_file_name)
-
     try:
-        with io.open(dump_file_path, 'w', encoding='utf-8') as outfile:
+        with io.open(json_file_abs_path, 'w', encoding='utf-8') as outfile:
             if is_py2:
                 outfile.write(
                     unicode(json.dumps(
@@ -645,23 +639,44 @@ def dump_json_file(json_data, pwd_dir_path, dump_file_name):
                     cls=PythonObjectEncoder
                 )
 
-        msg = "dump file: {}".format(dump_file_path)
+        msg = "dump file: {}".format(json_file_abs_path)
         logger.color_print(msg, "BLUE")
 
     except TypeError as ex:
-        msg = "Failed to dump json file: {}\nReason: {}".format(dump_file_path, ex)
+        msg = "Failed to dump json file: {}\nReason: {}".format(json_file_abs_path, ex)
         logger.color_print(msg, "RED")
 
 
-def _prepare_dump_info(project_mapping, tag_name):
+def prepare_dump_json_file_abs_path(project_mapping, tag_name):
-    """ prepare dump file info.
+    """ prepare dump json file absolute path.
     """
-    test_path = project_mapping.get("test_path") or "tests_mapping"
     pwd_dir_path = project_mapping.get("PWD") or os.getcwd()
-    file_name, file_suffix = os.path.splitext(os.path.basename(test_path.rstrip("/")))
+    test_path = project_mapping.get("test_path")
-    dump_file_name = "{}.{}.json".format(file_name, tag_name)
 
-    return pwd_dir_path, dump_file_name
+    if not test_path:
+        # running passed in testcase/testsuite data structure
+        dump_file_name = "tests_mapping.{}.json".format(tag_name)
+        dumped_json_file_abs_path = os.path.join(pwd_dir_path, "logs", dump_file_name)
+        return dumped_json_file_abs_path
+
+    # both test_path and pwd_dir_path are absolute path
+    logs_dir_path = os.path.join(pwd_dir_path, "logs")
+    test_path_relative_path = test_path[len(pwd_dir_path)+1:]
+
+    if os.path.isdir(test_path):
+        file_foder_path = os.path.join(logs_dir_path, test_path_relative_path)
+        dump_file_name = "all.{}.json".format(tag_name)
+    else:
+        file_relative_folder_path, test_file = os.path.split(test_path_relative_path)
+        file_foder_path = os.path.join(logs_dir_path, file_relative_folder_path)
+        test_file_name, _file_suffix = os.path.splitext(test_file)
+        dump_file_name = "{}.{}.json".format(test_file_name, tag_name)
+
+    if not os.path.isdir(file_foder_path):
+        os.makedirs(file_foder_path)
+
+    dumped_json_file_abs_path = os.path.join(file_foder_path, dump_file_name)
+    return dumped_json_file_abs_path
 
 
 def dump_logs(json_data, project_mapping, tag_name):
@@ -674,8 +689,8 @@ def dump_logs(json_data, project_mapping, tag_name):
         tag_name (str): tag name, loaded/parsed/summary
 
     """
-    pwd_dir_path, dump_file_name = _prepare_dump_info(project_mapping, tag_name)
+    json_file_abs_path = prepare_dump_json_file_abs_path(project_mapping, tag_name)
-    dump_json_file(json_data, pwd_dir_path, dump_file_name)
+    dump_json_file(json_data, json_file_abs_path)
 
 
 def get_python2_retire_msg():

httprunner/validator.py

@@ -1,137 +1,8 @@
 # encoding: utf-8
 import collections
-import io
 import json
-import os
-import types
 
-from httprunner import exceptions, logger
+from httprunner import exceptions, logger, parser
-
-""" validate data format
-TODO: refactor with JSON schema validate
-"""
-
-
-def is_testcase(data_structure):
-    """ check if data_structure is a testcase.
-
-    Args:
-        data_structure (dict): testcase should always be in the following data structure:
-
-            {
-                "config": {
-                    "name": "desc1",
-                    "variables": [],    # optional
-                    "request": {}       # optional
-                },
-                "teststeps": [
-                    test_dict1,
-                    {   # test_dict2
-                        'name': 'test step desc2',
-                        'variables': [],    # optional
-                        'extract': [],      # optional
-                        'validate': [],
-                        'request': {},
-                        'function_meta': {}
-                    }
-                ]
-            }
-
-    Returns:
-        bool: True if data_structure is valid testcase, otherwise False.
-
-    """
-    # TODO: replace with JSON schema validation
-    if not isinstance(data_structure, dict):
-        return False
-
-    if "teststeps" not in data_structure:
-        return False
-
-    if not isinstance(data_structure["teststeps"], list):
-        return False
-
-    return True
-
-
-def is_testcases(data_structure):
-    """ check if data_structure is testcase or testcases list.
-
-    Args:
-        data_structure (dict): testcase(s) should always be in the following data structure:
-            {
-                "project_mapping": {
-                    "PWD": "XXXXX",
-                    "functions": {},
-                    "env": {}
-                },
-                "testcases": [
-                    {   # testcase data structure
-                        "config": {
-                            "name": "desc1",
-                            "path": "testcase1_path",
-                            "variables": [],                    # optional
-                        },
-                        "teststeps": [
-                            # test data structure
-                            {
-                                'name': 'test step desc1',
-                                'variables': [],    # optional
-                                'extract': [],      # optional
-                                'validate': [],
-                                'request': {}
-                            },
-                            test_dict_2   # another test dict
-                        ]
-                    },
-                    testcase_dict_2     # another testcase dict
-                ]
-            }
-
-    Returns:
-        bool: True if data_structure is valid testcase(s), otherwise False.
-
-    """
-    if not isinstance(data_structure, dict):
-        return False
-
-    if "testcases" not in data_structure:
-        return False
-
-    testcases = data_structure["testcases"]
-    if not isinstance(testcases, list):
-        return False
-
-    for item in testcases:
-        if not is_testcase(item):
-            return False
-
-    return True
-
-
-def is_testcase_path(path):
-    """ check if path is testcase path or path list.
-
-    Args:
-        path (str/list): file path or file path list.
-
-    Returns:
-        bool: True if path is valid file path or path list, otherwise False.
-
-    """
-    if not isinstance(path, (str, list)):
-        return False
-
-    if isinstance(path, list):
-        for p in path:
-            if not is_testcase_path(p):
-                return False
-
-    if isinstance(path, str):
-        if not os.path.exists(path):
-            return False
-
-    return True
 
 
 ###############################################################################
@@ -159,13 +30,13 @@ def get_uniform_comparator(comparator):
         return "length_equals"
     elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]:
         return "length_greater_than"
-    elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \
+    elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals",
-        "count_greater_than_or_equals"]:
+                        "count_greater_than_or_equals"]:
         return "length_greater_than_or_equals"
     elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]:
         return "length_less_than"
-    elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \
+    elif comparator in ["len_le", "count_le", "length_less_than_or_equals",
-        "count_less_than_or_equals"]:
+                        "count_less_than_or_equals"]:
         return "length_less_than_or_equals"
     else:
         return comparator
@@ -237,14 +108,14 @@ def _convert_validators_to_mapping(validators):
 
     Examples:
         >>> validators = [
-                {"check": "v1", "expect": 201, "comparator": "eq"},
+            {"check": "v1", "expect": 201, "comparator": "eq"},
-                {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
+            {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
-            ]
+        ]
-        >>> _convert_validators_to_mapping(validators)
+        >>> print(_convert_validators_to_mapping(validators))
-            {
+        {
-                ("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
+            ("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
-                ('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
+            ('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
-            }
+        }
 
     """
     validators_mapping = {}
@@ -298,50 +169,189 @@ def extend_validators(raw_validators, override_validators):
         return list(def_validators_mapping.values())
 
 
-###############################################################################
+class Validator(object):
-##   validate varibles and functions
+    """Validate tests
-###############################################################################
 
+    Attributes:
+        validation_results (dict): store validation results,
+            including validate_extractor and validate_script.
 
-def is_function(item):
-    """ Takes item object, returns True if it is a function.
     """
-    return isinstance(item, types.FunctionType)
 
+    def __init__(self, session_context, resp_obj):
+        """ initialize a Validator for each teststep (API request)
 
-def is_variable(tup):
+        Args:
-    """ Takes (name, object) tuple, returns True if it is a variable.
+            session_context: HttpRunner session context
-    """
+            resp_obj: ResponseObject instance
-    name, item = tup
+        """
-    if callable(item):
+        self.session_context = session_context
-        # function or class
+        self.resp_obj = resp_obj
-        return False
+        self.validation_results = {}
 
-    if isinstance(item, types.ModuleType):
+    def __eval_validator_check(self, check_item):
-        # imported module
+        """ evaluate check item in validator.
-        return False
 
-    if name.startswith("_"):
+        Args:
-        # private property
+            check_item: check_item should only be the following 5 formats:
-        return False
+                1, variable reference, e.g. $token
+                2, function reference, e.g. ${is_status_code_200($status_code)}
+                3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
+                4, string joined by delimiter. e.g. "status_code", "headers.content-type"
+                5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
 
-    return True
+        """
+        if isinstance(check_item, (dict, list)) \
+                or isinstance(check_item, parser.LazyString):
+            # format 1/2/3
+            check_value = self.session_context.eval_content(check_item)
+        else:
+            # format 4/5
+            check_value = self.resp_obj.extract_field(check_item)
 
+        return check_value
 
-def validate_json_file(file_list):
+    def __eval_validator_expect(self, expect_item):
-    """ validate JSON testcase format
+        """ evaluate expect item in validator.
-    """
-    for json_file in set(file_list):
-        if not json_file.endswith(".json"):
-            logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file))
-            continue
 
-        logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN")
+        Args:
+            expect_item: expect_item should only be in 2 types:
+                1, variable reference, e.g. $expect_status_code
+                2, actual value, e.g. 200
+
+        """
+        expect_value = self.session_context.eval_content(expect_item)
+        return expect_value
+
+    def validate_script(self, script):
+        """ make validation with python script
+        """
+        validator_dict = {
+            "validate_script": "<br/>".join(script),
+            "check_result": "fail",
+            "exception": ""
+        }
+
+        script = "\n    ".join(script)
+        code = """
+# encoding: utf-8
+
+try:
+    {}
+except Exception as ex:
+    import traceback
+    import sys
+    _type, _value, _tb = sys.exc_info()
+    # filename, lineno, name, line
+    _, _lineno, _, line_content = traceback.extract_tb(_tb, 1)[0]
+
+    line_no = _lineno - 4
+
+    c_exception = _type.__name__ + "\\n"
+    c_exception += "\\tError line number: " + str(line_no) + "\\n"
+    c_exception += "\\tError line content: " + str(line_content) + "\\n"
+
+    if _value.args:
+        c_exception += "\\tError description: " + str(_value)
+    else:
+        c_exception += "\\tError description: " + _type.__name__
+
+    raise _type(c_exception)
+""".format(script)
+        variables = {
+            "status_code": self.resp_obj.status_code,
+            "response_json": self.resp_obj.json,
+            "response": self.resp_obj
+        }
+        variables.update(self.session_context.test_variables_mapping)
+
+        try:
+            code = compile(code, '<string>', 'exec')
+            exec(code, variables)
+            validator_dict["check_result"] = "pass"
+            return validator_dict, ""
+        except Exception as ex:
+            validator_dict["check_result"] = "fail"
+            validator_dict["exception"] = "<br/>".join(str(ex).splitlines())
+            return validator_dict, str(ex)
+
+    def validate(self, validators):
+        """ make validation with comparators
+        """
+        self.validation_results = {}
+        if not validators:
+            return
+
+        logger.log_debug("start to validate.")
+
+        validate_pass = True
+        failures = []
+
+        for validator in validators:
+
+            if isinstance(validator, dict) and validator.get("type") == "python_script":
+                validator_dict, ex = self.validate_script(validator["script"])
+                if ex:
+                    validate_pass = False
+                    failures.append(ex)
+
+                self.validation_results["validate_script"] = validator_dict
+                continue
+
+            if "validate_extractor" not in self.validation_results:
+                self.validation_results["validate_extractor"] = []
+
+            # validator should be LazyFunction object
+            if not isinstance(validator, parser.LazyFunction):
+                raise exceptions.ValidationFailure(
+                    "validator should be parsed first: {}".format(validators))
+
+            # evaluate validator args with context variable mapping.
+            validator_args = validator.get_args()
+            check_item, expect_item = validator_args
+            check_value = self.__eval_validator_check(check_item)
+            expect_value = self.__eval_validator_expect(expect_item)
+            validator.update_args([check_value, expect_value])
+
+            comparator = validator.func_name
+            validator_dict = {
+                "comparator": comparator,
+                "check": check_item,
+                "check_value": check_value,
+                "expect": expect_item,
+                "expect_value": expect_value
+            }
+            validate_msg = "\nvalidate: {} {} {}({})".format(
+                check_item,
+                comparator,
+                expect_value,
+                type(expect_value).__name__
+            )
 
-        with io.open(json_file) as stream:
             try:
-                json.load(stream)
+                validator.to_value(self.session_context.test_variables_mapping)
-            except ValueError as e:
+                validator_dict["check_result"] = "pass"
-                raise SystemExit(e)
+                validate_msg += "\t==> pass"
+                logger.log_debug(validate_msg)
+            except (AssertionError, TypeError):
+                validate_pass = False
+                validator_dict["check_result"] = "fail"
+                validate_msg += "\t==> fail"
+                validate_msg += "\n{}({}) {} {}({})".format(
+                    check_value,
+                    type(check_value).__name__,
+                    comparator,
+                    expect_value,
+                    type(expect_value).__name__
+                )
+                logger.log_error(validate_msg)
+                failures.append(validate_msg)
 
-        print("OK")
+            self.validation_results["validate_extractor"].append(validator_dict)
+
+            # restore validator args, in case of running multiple times
+            validator.update_args(validator_args)
+
+        if not validate_pass:
+            failures_string = "\n".join([failure for failure in failures])
+            raise exceptions.ValidationFailure(failures_string)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "httprunner"
-version = "2.3.2"
+version = "2.4.0"
 description = "One-stop solution for HTTP(S) testing."
 license = "Apache-2.0"
 readme = "README.md"

tests/debugtalk.py

@@ -127,6 +127,15 @@ def alter_response(response):
         "key": 123
     }
 
+def alter_response_302(response):
+    response.status_code = 500
+    response.headers["Content-Type"] = "html/text"
+    response.text = "abcdef"
+    response.new_attribute = "new_attribute_value"
+    response.new_attribute_dict = {
+        "key": 123
+    }
+
 
 def alter_response_error(response):
     # NameError

tests/httpbin/a.b.c/rpc.yml

@@ -0,0 +1,10 @@
+name: rpc api
+base_url: http://httpbin.org
+variables:
+    expected_status_code: 200
+request:
+    url: /headers
+    method: GET
+validate:
+    - eq: ["status_code", $expected_status_code]
+    - eq: [content.headers.Host, "httpbin.org"]

tests/httpbin/api/302_redirect_teardown_hook.yml

@@ -0,0 +1,13 @@
+name: 302 redirect
+request:
+    url: https://httpbin.org/redirect-to
+    params:
+      url: https://github.com
+      status_code: 302
+    method: GET
+    verify: False
+teardown_hooks:
+  - ${alter_response_302($response)}
+validate:
+    - eq: ["status_code", 500]
+    - eq: ["text","abcdef"]

tests/httpbin/validate.yml

@@ -1,13 +1,34 @@
 - config:
     name: basic test with httpbin
-    request:
+    base_url: http://httpbin.org/
-        base_url: http://httpbin.org/
 
 - test:
-    name: headers
+    name: validate response with json path
     request:
-        url: /headers
+        url: /get
+        params:
+            a: 1
+            b: 2
         method: GET
     validate:
         - eq: ["status_code", 200]
-        - assert_status_code_is_200: ["status_code"]
+        - eq: ["json.args.a", '1']
+        - eq: ["json.args.b", '2']
+    validate_script:
+        - "assert status_code == 200"
+
+
+- test:
+    name: validate response with python script
+    request:
+        url: /get
+        params:
+            a: 1
+            b: 2
+        method: GET
+    validate:
+        - eq: ["status_code", 200]
+    validate_script:
+        - "assert status_code == 201"
+        - "a = response_json.get('args').get('a')"
+        - "assert a == '1'"

tests/test_api.py

@@ -289,6 +289,10 @@ class TestHttpRunner(ApiServerUnittest):
         self.assertEqual(summary["stat"]["testcases"]["total"], 2)
         self.assertEqual(summary["stat"]["teststeps"]["total"], 4)
 
+    def test_validate_script(self):
+        summary = self.runner.run("tests/httpbin/validate.yml")
+        self.assertFalse(summary["success"])
+
     def test_run_httprunner_with_hooks(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/httpbin/hooks.yml')
@@ -419,6 +423,19 @@ class TestHttpRunner(ApiServerUnittest):
         self.assertEqual(req_resp_data[0]["response"]["status_code"], 302)
         self.assertEqual(req_resp_data[1]["response"]["status_code"], 200)
 
+    def test_request_302_logs_teardown_hook(self):
+        path = "tests/httpbin/api/302_redirect_teardown_hook.yml"
+        summary = self.runner.run(path)
+        self.assertTrue(summary["success"])
+        self.assertEqual(summary["stat"]["testcases"]["total"], 1)
+        self.assertEqual(summary["stat"]["teststeps"]["total"], 1)
+        self.assertEqual(summary["stat"]["teststeps"]["successes"], 1)
+
+        req_resp_data = summary["details"][0]["records"][0]["meta_datas"]["data"]
+        self.assertEqual(len(req_resp_data), 2)
+        self.assertEqual(req_resp_data[0]["response"]["status_code"], 302)
+        self.assertEqual(req_resp_data[1]["response"]["status_code"], 500)
+
     def test_request_with_params(self):
         path = "tests/httpbin/api/302_redirect.yml"
         summary = self.runner.run(path)
@@ -437,13 +454,17 @@ class TestHttpRunner(ApiServerUnittest):
     def test_run_api_folder(self):
         api_folder = "tests/httpbin/api/"
         summary = self.runner.run(api_folder)
+        print(summary["stat"]["testcases"]["total"])
+        print(len(summary["details"]))
         self.assertTrue(summary["success"])
-        self.assertEqual(summary["stat"]["testcases"]["total"], 2)
+        self.assertEqual(summary["stat"]["testcases"]["total"], 3)
-        self.assertEqual(summary["stat"]["teststeps"]["total"], 2)
+        self.assertEqual(summary["stat"]["teststeps"]["total"], 3)
-        self.assertEqual(summary["stat"]["teststeps"]["successes"], 2)
+        self.assertEqual(summary["stat"]["teststeps"]["successes"], 3)
-        self.assertEqual(len(summary["details"]), 2)
+        self.assertEqual(len(summary["details"]), 3)
         self.assertEqual(summary["details"][0]["stat"]["total"], 1)
         self.assertEqual(summary["details"][1]["stat"]["total"], 1)
+        self.assertEqual(summary["details"][2]["stat"]["total"], 1)
+
 
     def test_run_testcase_hardcode(self):
         for testcase_file_path in self.testcase_file_path_list:

tests/test_loader.py

@@ -476,3 +476,73 @@ class TestSuiteLoader(unittest.TestCase):
         api_file_path = os.path.join(os.getcwd(), "tests", "api", "get_token.yml")
         self.assertIn(api_file_path, self.tests_def_mapping["api"])
         self.assertEqual(self.project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH")
+
+    def test_is_function(self):
+        func = lambda x: x + 1
+        self.assertTrue(loader.is_function(func))
+        self.assertTrue(loader.is_function(loader.is_testcase))
+
+    def test_is_testcases(self):
+        data_structure = "path/to/file"
+        self.assertFalse(loader.is_testcases(data_structure))
+        data_structure = ["path/to/file1", "path/to/file2"]
+        self.assertFalse(loader.is_testcases(data_structure))
+
+        data_structure = {
+            "project_mapping": {
+                "PWD": "XXXXX",
+                "functions": {},
+                "env": {}
+            },
+            "testcases": [
+                {   # testcase data structure
+                    "config": {
+                        "name": "desc1",
+                        "path": "testcase1_path",
+                        "variables": [],                    # optional
+                    },
+                    "teststeps": [
+                        # test data structure
+                        {
+                            'name': 'test step desc1',
+                            'variables': [],    # optional
+                            'extract': [],      # optional
+                            'validate': [],
+                            'request': {}
+                        },
+                        # test_dict2   # another test dict
+                    ]
+                },
+                # testcase_dict_2     # another testcase dict
+            ]
+        }
+        self.assertTrue(loader.is_testcases(data_structure))
+        data_structure = [
+            {
+                "name": "desc1",
+                "config": {},
+                "api": {},
+                "testcases": ["testcase11", "testcase12"]
+            },
+            {
+                "name": "desc2",
+                "config": {},
+                "api": {},
+                "testcases": ["testcase21", "testcase22"]
+            }
+        ]
+        self.assertTrue(data_structure)
+
+    def test_is_variable(self):
+        var1 = 123
+        var2 = "abc"
+        self.assertTrue(loader.is_variable(("var1", var1)))
+        self.assertTrue(loader.is_variable(("var2", var2)))
+
+        __var = 123
+        self.assertFalse(loader.is_variable(("__var", __var)))
+
+        func = lambda x: x + 1
+        self.assertFalse(loader.is_variable(("func", func)))
+
+        self.assertFalse(loader.is_variable(("unittest", unittest)))

tests/test_utils.py

@@ -275,3 +275,37 @@ class TestUtils(ApiServerUnittest):
             "d": [4, 5]
         }
         utils.print_info(info_mapping)
+
+    def test_prepare_dump_json_file_path_for_folder(self):
+        # hrun tests/httpbin/a.b.c/ --save-tests
+        project_working_directory = os.path.join(os.getcwd(), "tests")
+        project_mapping = {
+            "PWD": project_working_directory,
+            "test_path": os.path.join(os.getcwd(), "tests", "httpbin", "a.b.c")
+        }
+        self.assertEqual(
+            utils.prepare_dump_json_file_abs_path(project_mapping, "loaded"),
+            os.path.join(project_working_directory, "logs", "httpbin/a.b.c/all.loaded.json")
+        )
+
+    def test_prepare_dump_json_file_path_for_file(self):
+        # hrun tests/httpbin/a.b.c/rpc.yml --save-tests
+        project_working_directory = os.path.join(os.getcwd(), "tests")
+        project_mapping = {
+            "PWD": project_working_directory,
+            "test_path": os.path.join(os.getcwd(), "tests", "httpbin", "a.b.c", "rpc.yml")
+        }
+        self.assertEqual(
+            utils.prepare_dump_json_file_abs_path(project_mapping, "loaded"),
+            os.path.join(project_working_directory, "logs", "httpbin/a.b.c/rpc.loaded.json")
+        )
+
+    def test_prepare_dump_json_file_path_for_passed_testcase(self):
+        project_working_directory = os.path.join(os.getcwd(), "tests")
+        project_mapping = {
+            "PWD": project_working_directory
+        }
+        self.assertEqual(
+            utils.prepare_dump_json_file_abs_path(project_mapping, "loaded"),
+            os.path.join(project_working_directory, "logs", "tests_mapping.loaded.json")
+        )

tests/test_validator.py

@@ -5,76 +5,6 @@ from httprunner import validator
 
 class TestValidator(unittest.TestCase):
 
-    def test_is_testcases(self):
-        data_structure = "path/to/file"
-        self.assertFalse(validator.is_testcases(data_structure))
-        data_structure = ["path/to/file1", "path/to/file2"]
-        self.assertFalse(validator.is_testcases(data_structure))
-
-        data_structure = {
-            "project_mapping": {
-                "PWD": "XXXXX",
-                "functions": {},
-                "env": {}
-            },
-            "testcases": [
-                {   # testcase data structure
-                    "config": {
-                        "name": "desc1",
-                        "path": "testcase1_path",
-                        "variables": [],                    # optional
-                    },
-                    "teststeps": [
-                        # test data structure
-                        {
-                            'name': 'test step desc1',
-                            'variables': [],    # optional
-                            'extract': [],      # optional
-                            'validate': [],
-                            'request': {}
-                        },
-                        # test_dict2   # another test dict
-                    ]
-                },
-                # testcase_dict_2     # another testcase dict
-            ]
-        }
-        self.assertTrue(validator.is_testcases(data_structure))
-        data_structure = [
-            {
-                "name": "desc1",
-                "config": {},
-                "api": {},
-                "testcases": ["testcase11", "testcase12"]
-            },
-            {
-                "name": "desc2",
-                "config": {},
-                "api": {},
-                "testcases": ["testcase21", "testcase22"]
-            }
-        ]
-        self.assertTrue(data_structure)
-
-    def test_is_variable(self):
-        var1 = 123
-        var2 = "abc"
-        self.assertTrue(validator.is_variable(("var1", var1)))
-        self.assertTrue(validator.is_variable(("var2", var2)))
-
-        __var = 123
-        self.assertFalse(validator.is_variable(("__var", __var)))
-
-        func = lambda x: x + 1
-        self.assertFalse(validator.is_variable(("func", func)))
-
-        self.assertFalse(validator.is_variable(("unittest", unittest)))
-
-    def test_is_function(self):
-        func = lambda x: x + 1
-        self.assertTrue(validator.is_function(func))
-        self.assertTrue(validator.is_function(validator.is_testcase))
-
     def test_get_uniform_comparator(self):
         self.assertEqual(validator.get_uniform_comparator("eq"), "equals")
         self.assertEqual(validator.get_uniform_comparator("=="), "equals")
@@ -117,7 +47,6 @@ class TestValidator(unittest.TestCase):
             {"check": "status_code", "comparator": "equals", "expect": 201}
         )
 
-
     def test_extend_validators(self):
         def_validators = [
             {'eq': ['v1', 200]},