validate all validators even if failed

httprunner/__about__.py

@@ -1,7 +1,7 @@
 __title__ = 'HttpRunner'
 __description__ = 'One-stop solution for HTTP(S) testing.'
 __url__ = 'https://github.com/HttpRunner/HttpRunner'
-__version__ = '1.5.6'
+__version__ = '1.5.7'
 __author__ = 'debugtalk'
 __author_email__ = 'mail@debugtalk.com'
 __license__ = 'MIT'

httprunner/context.py

@@ -5,7 +5,7 @@ import os
 import re
 import sys
 
-from httprunner import exception, testcase, utils
+from httprunner import exception, logger, testcase, utils
 from httprunner.compat import OrderedDict
 
 
@@ -237,32 +237,50 @@ class Context(object):
             and comparator not in ["is", "eq", "equals", "=="]:
             raise exception.ParamsError("Null value can only be compared with comparator: eq/equals/==")
 
-        try:
+        validate_msg = "validate: {} {} {}({})".format(
-            validator_dict["check_result"] = "passed"
+            check_item,
-            validate_func(validator_dict["check_value"], validator_dict["expect"])
+            comparator,
-        except (AssertionError, TypeError):
+            expect_value,
-            err_msg = "\n" + "\n".join([
+            type(expect_value).__name__
-                "\tcheck item name: %s;" % check_item,
+        )
-                "\tcheck item value: %s (%s);" % (check_value, type(check_value).__name__),
-                "\tcomparator: %s;" % comparator,
-                "\texpected value: %s (%s)." % (expect_value, type(expect_value).__name__)
-            ])
-            validator_dict["check_result"] = "failed"
-            raise exception.ValidationError(err_msg)
 
-    def eval_validators(self, validators, resp_obj):
+        try:
-        """ evaluate validators with context variable mapping.
+            validator_dict["check_result"] = "pass"
+            validate_func(check_value, expect_value)
+            validate_msg += "\t==> pass"
+            logger.log_debug(validate_msg)
+        except (AssertionError, TypeError):
+            validate_msg += "\t==> fail"
+            validate_msg += "\n{}({}) {} {}({})".format(
+                check_value,
+                type(check_value).__name__,
+                comparator,
+                expect_value,
+                type(expect_value).__name__
+            )
+            logger.log_error(validate_msg)
+            validator_dict["check_result"] = "fail"
+            raise exception.ValidationError(validate_msg)
+
+    def validate(self, validators, resp_obj):
+        """ make validations
         """
-        return [
+        self.evaluated_validators = []
-            self.eval_check_item(
+        validate_pass = True
+
+        for validator in validators:
+            # evaluate validators with context variable mapping.
+            evaluated_validator = self.eval_check_item(
                 testcase.parse_validator(validator),
                 resp_obj
             )
-            for validator in validators
-        ]
 
-    def validate(self, validators):
+            try:
-        """ make validations
+                self.do_validation(evaluated_validator)
-        """
+            except exception.ValidationError:
-        for validator_dict in validators:
+                validate_pass = False
-            self.do_validation(validator_dict)
+
+            self.evaluated_validators.append(evaluated_validator)
+
+        if not validate_pass:
+            raise exception.ValidationError

httprunner/runner.py

@@ -11,7 +11,6 @@ class Runner(object):
 
     def __init__(self, config_dict=None, http_client_session=None):
         self.http_client_session = http_client_session
-        self.evaluated_validators = []
         self.context = Context()
 
         config_dict = config_dict or {}
@@ -183,8 +182,7 @@ class Runner(object):
         # validate
         validators = testcase_dict.get("validate", []) or testcase_dict.get("validators", [])
         try:
-            self.evaluated_validators = self.context.eval_validators(validators, resp_obj)
+            self.context.validate(validators, resp_obj)
-            self.context.validate(self.evaluated_validators)
         except (exception.ParamsError, exception.ResponseError, \
             exception.ValidationError, exception.ParseResponseError):
             # log request

httprunner/task.py

@@ -28,7 +28,7 @@ class TestCase(unittest.TestCase):
         finally:
             if hasattr(self.test_runner.http_client_session, "meta_data"):
                 self.meta_data = self.test_runner.http_client_session.meta_data
-                self.meta_data["validators"] = self.test_runner.evaluated_validators
+                self.meta_data["validators"] = self.test_runner.context.evaluated_validators
                 self.test_runner.http_client_session.init_meta_data()
 
 

httprunner/templates/default_report_template.html

@@ -307,9 +307,9 @@
                       </tr>
                       {% for validator in record.meta_data.validators %}
                       <tr>
-                        {% if validator.check_result == "passed" %}
+                        {% if validator.check_result == "pass" %}
                         <td class="passed">
-                        {% elif validator.check_result == "failed" %}
+                        {% elif validator.check_result == "fail" %}
                         <td class="failed">
                         {% elif validator.check_result == "unchecked" %}
                         <td class="unchecked">

tests/test_context.py

@@ -271,8 +271,7 @@ class VariableBindsUnittest(ApiServerUnittest):
         self.context.bind_variables(variables)
 
         with self.assertRaises(exception.ValidationError):
-            evaluated_validators = self.context.eval_validators(validators, resp_obj)
+            self.context.validate(validators, resp_obj)
-            self.context.validate(evaluated_validators)
 
         validators = [
             {"eq": ["$resp_status_code", 201]},
@@ -291,8 +290,7 @@ class VariableBindsUnittest(ApiServerUnittest):
         }
         self.context.bind_functions(functions)
 
-        evaluated_validators = self.context.eval_validators(validators, resp_obj)
+        self.context.validate(validators, resp_obj)
-        self.context.validate(evaluated_validators)
 
     def test_validate_exception(self):
         url = "http://127.0.0.1:5000/"
@@ -308,8 +306,7 @@ class VariableBindsUnittest(ApiServerUnittest):
         self.context.bind_variables(variables)
 
         with self.assertRaises(exception.ParamsError):
-            evaluated_validators = self.context.eval_validators(validators, resp_obj)
+            self.context.validate(validators, resp_obj)
-            self.context.validate(evaluated_validators)
 
         # expected value missed in variables mapping
         variables = [
@@ -318,5 +315,4 @@ class VariableBindsUnittest(ApiServerUnittest):
         self.context.bind_variables(variables)
 
         with self.assertRaises(exception.ValidationError):
-            evaluated_validators = self.context.eval_validators(validators, resp_obj)
+            self.context.validate(validators, resp_obj)
-            self.context.validate(evaluated_validators)