diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 4741e330..80ebd1fb 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.5.0' +__version__ = '1.5.1' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/context.py b/httprunner/context.py index 77170155..9decb7ca 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -216,6 +216,7 @@ class Context(object): # 2, actual value, e.g. 200 expect_value = self.eval_content(validator["expect"]) validator["expect"] = expect_value + validator["check_result"] = "unchecked" return validator def do_validation(self, validator_dict): @@ -237,6 +238,7 @@ class Context(object): raise exception.ParamsError("Null value can only be compared with comparator: eq/equals/==") try: + validator_dict["check_result"] = "passed" validate_func(validator_dict["check_value"], validator_dict["expect"]) except (AssertionError, TypeError): err_msg = "\n" + "\n".join([ @@ -245,18 +247,22 @@ class Context(object): "\tcomparator: %s;" % comparator, "\texpected value: %s (%s)." % (expect_value, type(expect_value).__name__) ]) + validator_dict["check_result"] = "failed" raise exception.ValidationError(err_msg) - def validate(self, validators, resp_obj): - """ check validators with the context variable mapping. - @param (list) validators - @param (object) resp_obj + def eval_validators(self, validators, resp_obj): + """ evaluate validators with context variable mapping. """ - for validator in validators: - validator_dict = self.eval_check_item( + return [ + self.eval_check_item( testcase.parse_validator(validator), resp_obj ) - self.do_validation(validator_dict) + for validator in validators + ] - return True + def validate(self, validators): + """ make validations + """ + for validator_dict in validators: + self.do_validation(validator_dict) diff --git a/httprunner/runner.py b/httprunner/runner.py index 00919d12..fbf8031f 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -11,6 +11,7 @@ class Runner(object): def __init__(self, config_dict=None, http_client_session=None): self.http_client_session = http_client_session + self.evaluated_validators = [] self.context = Context() config_dict = config_dict or {} @@ -178,11 +179,12 @@ class Runner(object): extractors = testcase_dict.get("extract", []) or testcase_dict.get("extractors", []) extracted_variables_mapping = resp_obj.extract_response(extractors) self.context.bind_extracted_variables(extracted_variables_mapping) - + # validate validators = testcase_dict.get("validate", []) or testcase_dict.get("validators", []) try: - self.context.validate(validators, resp_obj) + self.evaluated_validators = self.context.eval_validators(validators, resp_obj) + self.context.validate(self.evaluated_validators) except (exception.ParamsError, exception.ResponseError, \ exception.ValidationError, exception.ParseResponseError): # log request diff --git a/httprunner/task.py b/httprunner/task.py index 76f28fa0..62bce835 100644 --- a/httprunner/task.py +++ b/httprunner/task.py @@ -28,6 +28,7 @@ class TestCase(unittest.TestCase): finally: if hasattr(self.test_runner.http_client_session, "meta_data"): self.meta_data = self.test_runner.http_client_session.meta_data + self.meta_data["validators"] = self.test_runner.evaluated_validators self.test_runner.http_client_session.init_meta_data() diff --git a/httprunner/templates/default_report_template.html b/httprunner/templates/default_report_template.html index 9f9f8b38..a2ddcd22 100644 --- a/httprunner/templates/default_report_template.html +++ b/httprunner/templates/default_report_template.html @@ -30,6 +30,15 @@ background-color: skyblue; padding: 5px 12px; } + .details tr .passed { + background-color: lightgreen; + } + .details tr .failed { + background-color: red; + } + .details tr .unchecked { + background-color: gray; + } .details td { background-color: lightblue; padding: 5px 12px; @@ -277,6 +286,34 @@ +

Validators:

+
+ + + + + + + + {% for validator in record.meta_data.validators %} + + {% if validator.check_result == "passed" %} + + + + + + {% endfor %} +
checkcomparatorexpect valueactual value
+ {% elif validator.check_result == "failed" %} + + {% elif validator.check_result == "unchecked" %} + + {% endif %} + {{validator.check}} + {{validator.comparator}}{{validator.expect}}{{validator.check_value}}
+
+

Statistics:

diff --git a/tests/test_context.py b/tests/test_context.py index 647390bc..746c79e3 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -262,8 +262,7 @@ class VariableBindsUnittest(ApiServerUnittest): validators = [ {"eq": ["$resp_status_code", 201]}, {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, - {"check": "$resp_body_success", "comparator": "eq", "expect": True}, - {"check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False} + {"check": "$resp_body_success", "comparator": "eq", "expect": True} ] variables = [ {"resp_status_code": 200}, @@ -272,8 +271,15 @@ class VariableBindsUnittest(ApiServerUnittest): self.context.bind_variables(variables) with self.assertRaises(exception.ValidationError): - self.context.validate(validators, resp_obj) + evaluated_validators = self.context.eval_validators(validators, resp_obj) + self.context.validate(evaluated_validators) + validators = [ + {"eq": ["$resp_status_code", 201]}, + {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, + {"check": "$resp_body_success", "comparator": "eq", "expect": True}, + {"check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False} + ] variables = [ {"resp_status_code": 201}, {"resp_body_success": True} @@ -285,7 +291,8 @@ class VariableBindsUnittest(ApiServerUnittest): } self.context.bind_functions(functions) - self.assertTrue(self.context.validate(validators, resp_obj)) + evaluated_validators = self.context.eval_validators(validators, resp_obj) + self.context.validate(evaluated_validators) def test_validate_exception(self): url = "http://127.0.0.1:5000/" @@ -295,14 +302,14 @@ class VariableBindsUnittest(ApiServerUnittest): # expected value missed in validators validators = [ {"eq": ["$resp_status_code", 201]}, - {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, - {"check": "$resp_body_success", "comparator": "eq", "expect": True} + {"check": "$resp_status_code", "comparator": "eq", "expect": 201} ] variables = [] self.context.bind_variables(variables) with self.assertRaises(exception.ParamsError): - self.context.validate(validators, resp_obj) + evaluated_validators = self.context.eval_validators(validators, resp_obj) + self.context.validate(evaluated_validators) # expected value missed in variables mapping variables = [ @@ -311,4 +318,5 @@ class VariableBindsUnittest(ApiServerUnittest): self.context.bind_variables(variables) with self.assertRaises(exception.ValidationError): - self.context.validate(validators, resp_obj) + evaluated_validators = self.context.eval_validators(validators, resp_obj) + self.context.validate(evaluated_validators)