diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 1a104309..7909396e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,10 @@ ## 3.1.1 (2020-06-21) +**Added** + +- feat: add optional message for assertion + **Fixed** - fix #942: type_match None diff --git a/docs/images/httprunner-step-request-validate.png b/docs/images/httprunner-step-request-validate.png index a837da8d..d3e1e433 100644 Binary files a/docs/images/httprunner-step-request-validate.png and b/docs/images/httprunner-step-request-validate.png differ diff --git a/docs/user/write_testcase.md b/docs/user/write_testcase.md index 3f839015..4b914470 100644 --- a/docs/user/write_testcase.md +++ b/docs/user/write_testcase.md @@ -175,10 +175,11 @@ Extract JSON response body with [jmespath][jmespath]. Extract JSON response body with [jmespath][jmespath] and validate with expected value. -> assert_XXX(jmes_path: Text, expected_value: Any) +> assert_XXX(jmes_path: Text, expected_value: Any, message: Text = "") - jmes_path: jmespath expression, refer to [JMESPath Tutorial][jmespath_tutorial] for more details - expected_value: the specified expected value, variable or function reference can also be used here +- message (optional): used to indicate assertion error reason The image below shows HttpRunner builtin validators. diff --git a/examples/postman_echo/request_methods/request_with_functions.yml b/examples/postman_echo/request_methods/request_with_functions.yml index 634875d8..36ab12b5 100644 --- a/examples/postman_echo/request_methods/request_with_functions.yml +++ b/examples/postman_echo/request_methods/request_with_functions.yml @@ -63,7 +63,7 @@ teststeps: Content-Type: "application/x-www-form-urlencoded" data: "foo1=$foo1&foo2=$foo2&foo3=$foo3" validate: - - eq: ["status_code", 200] + - eq: ["status_code", 200, "response status code should be 200"] - eq: ["body.form.foo1", "$expect_foo1"] - eq: ["body.form.foo2", "bar23"] - eq: ["body.form.foo3", "bar21"] diff --git a/examples/postman_echo/request_methods/request_with_functions_test.py b/examples/postman_echo/request_methods/request_with_functions_test.py index 4ed62183..99d3731f 100644 --- a/examples/postman_echo/request_methods/request_with_functions_test.py +++ b/examples/postman_echo/request_methods/request_with_functions_test.py @@ -73,7 +73,7 @@ class TestCaseRequestWithFunctions(HttpRunner): ) .with_data("foo1=$foo1&foo2=$foo2&foo3=$foo3") .validate() - .assert_equal("status_code", 200) + .assert_equal("status_code", 200, "response status code should be 200") .assert_equal("body.form.foo1", "$expect_foo1") .assert_equal("body.form.foo2", "bar23") .assert_equal("body.form.foo3", "bar21") diff --git a/httprunner/builtin/comparators.py b/httprunner/builtin/comparators.py index b81919b3..897112ee 100644 --- a/httprunner/builtin/comparators.py +++ b/httprunner/builtin/comparators.py @@ -3,72 +3,101 @@ Built-in validate comparators. """ import re +from typing import Text, Any, Union -def equal(check_value, expect_value): - assert check_value == expect_value +def equal(check_value: Any, expect_value: Any, message: Text = ""): + assert check_value == expect_value, message -def greater_than(check_value, expect_value): - assert check_value > expect_value +def greater_than( + check_value: Union[int, float], expect_value: Union[int, float], message: Text = "" +): + assert check_value > expect_value, message -def less_than(check_value, expect_value): - assert check_value < expect_value +def less_than( + check_value: Union[int, float], expect_value: Union[int, float], message: Text = "" +): + assert check_value < expect_value, message -def greater_or_equals(check_value, expect_value): - assert check_value >= expect_value +def greater_or_equals( + check_value: Union[int, float], expect_value: Union[int, float], message: Text = "" +): + assert check_value >= expect_value, message -def less_or_equals(check_value, expect_value): - assert check_value <= expect_value +def less_or_equals( + check_value: Union[int, float], expect_value: Union[int, float], message: Text = "" +): + assert check_value <= expect_value, message -def not_equal(check_value, expect_value): - assert check_value != expect_value +def not_equal(check_value: Any, expect_value: Any, message: Text = ""): + assert check_value != expect_value, message -def string_equals(check_value, expect_value): - assert str(check_value) == str(expect_value) +def string_equals(check_value: Text, expect_value: Any, message: Text = ""): + assert str(check_value) == str(expect_value), message -def length_equal(check_value, expect_value): - assert isinstance(expect_value, int) - assert len(check_value) == expect_value +def length_equal(check_value: Text, expect_value: int, message: Text = ""): + assert isinstance(expect_value, int), "expect_value should be int type" + assert len(check_value) == expect_value, message -def length_greater_than(check_value, expect_value): - assert isinstance(expect_value, (int, float)) - assert len(check_value) > expect_value +def length_greater_than( + check_value: Text, expect_value: Union[int, float], message: Text = "" +): + assert isinstance( + expect_value, (int, float) + ), "expect_value should be int/float type" + assert len(check_value) > expect_value, message -def length_greater_or_equals(check_value, expect_value): - assert isinstance(expect_value, (int, float)) - assert len(check_value) >= expect_value +def length_greater_or_equals( + check_value: Text, expect_value: Union[int, float], message: Text = "" +): + assert isinstance( + expect_value, (int, float) + ), "expect_value should be int/float type" + assert len(check_value) >= expect_value, message -def length_less_than(check_value, expect_value): - assert isinstance(expect_value, (int, float)) - assert len(check_value) < expect_value +def length_less_than( + check_value: Text, expect_value: Union[int, float], message: Text = "" +): + assert isinstance( + expect_value, (int, float) + ), "expect_value should be int/float type" + assert len(check_value) < expect_value, message -def length_less_or_equals(check_value, expect_value): - assert isinstance(expect_value, (int, float)) - assert len(check_value) <= expect_value +def length_less_or_equals( + check_value: Text, expect_value: Union[int, float], message: Text = "" +): + assert isinstance( + expect_value, (int, float) + ), "expect_value should be int/float type" + assert len(check_value) <= expect_value, message -def contains(check_value, expect_value): - assert isinstance(check_value, (list, tuple, dict, str, bytes)) - assert expect_value in check_value +def contains(check_value: Any, expect_value: Any, message: Text = ""): + assert isinstance( + check_value, (list, tuple, dict, str, bytes) + ), "expect_value should be list/tuple/dict/str/bytes type" + assert expect_value in check_value, message -def contained_by(check_value, expect_value): - assert isinstance(expect_value, (list, tuple, dict, str, bytes)) - assert check_value in expect_value +def contained_by(check_value: Any, expect_value: Any, message: Text = ""): + assert isinstance( + check_value, (list, tuple, dict, str, bytes) + ), "expect_value should be list/tuple/dict/str/bytes type" + assert check_value in expect_value, message -def type_match(check_value, expect_value): +def type_match(check_value: Any, expect_value: Any, message: Text = ""): def get_type(name): if isinstance(name, type): return name @@ -81,20 +110,20 @@ def type_match(check_value, expect_value): raise ValueError(name) if expect_value in ["None", "NoneType", None]: - assert check_value is None + assert check_value is None, message else: - assert type(check_value) == get_type(expect_value) + assert type(check_value) == get_type(expect_value), message -def regex_match(check_value, expect_value): - assert isinstance(expect_value, str) - assert isinstance(check_value, str) - assert re.match(expect_value, check_value) +def regex_match(check_value: Text, expect_value: Any, message: Text = ""): + assert isinstance(expect_value, str), "expect_value should be Text type" + assert isinstance(check_value, str), "check_value should be Text type" + assert re.match(expect_value, check_value), message -def startswith(check_value, expect_value): - assert str(check_value).startswith(str(expect_value)) +def startswith(check_value: Any, expect_value: Any, message: Text = ""): + assert str(check_value).startswith(str(expect_value)), message -def endswith(check_value, expect_value): - assert str(check_value).endswith(str(expect_value)) +def endswith(check_value: Text, expect_value: Any, message: Text = ""): + assert str(check_value).endswith(str(expect_value)), message diff --git a/httprunner/make.py b/httprunner/make.py index 9053c0c8..62f54305 100644 --- a/httprunner/make.py +++ b/httprunner/make.py @@ -310,7 +310,12 @@ def make_teststep_chain_style(teststep: Dict) -> Text: expect = validator["expect"] if isinstance(expect, Text): expect = f'"{expect}"' - step_info += f".assert_{assert_method}({check}, {expect})" + + message = validator["message"] + if message: + step_info += f".assert_{assert_method}({check}, {expect}, '{message}')" + else: + step_info += f".assert_{assert_method}({check}, {expect})" return f"Step({step_info})" diff --git a/httprunner/response.py b/httprunner/response.py index 1efa945d..76690e9c 100644 --- a/httprunner/response.py +++ b/httprunner/response.py @@ -80,6 +80,7 @@ def uniform_validator(validator): # format1 check_item = validator["check"] expect_value = validator["expect"] + message = validator.get("message", "") comparator = validator.get("comparator", "eq") elif len(validator) == 1: @@ -87,10 +88,16 @@ def uniform_validator(validator): comparator = list(validator.keys())[0] compare_values = validator[comparator] - if not isinstance(compare_values, list) or len(compare_values) != 2: + if not isinstance(compare_values, list) or len(compare_values) not in [2, 3]: raise ParamsError(f"invalid validator: {validator}") - check_item, expect_value = compare_values + check_item = compare_values[0] + expect_value = compare_values[1] + if len(compare_values) == 3: + message = compare_values[2] + else: + # len(compare_values) == 2 + message = "" else: raise ParamsError(f"invalid validator: {validator}") @@ -98,7 +105,12 @@ def uniform_validator(validator): # uniform comparator, e.g. lt => less_than, eq => equals assert_method = get_uniform_comparator(comparator) - return {"check": check_item, "expect": expect_value, "assert": assert_method} + return { + "check": check_item, + "expect": expect_value, + "assert": assert_method, + "message": message, + } class ResponseObject(object): @@ -193,6 +205,11 @@ class ResponseObject(object): # parse expected value with config/teststep/extracted variables expect_value = parse_data(expect_item, variables_mapping, functions_mapping) + # message + message = u_validator["message"] + # parse message with config/teststep/extracted variables + message = parse_data(message, variables_mapping, functions_mapping) + validate_msg = f"assert {check_item} {assert_method} {expect_value}({type(expect_value).__name__})" validator_dict = { @@ -201,14 +218,15 @@ class ResponseObject(object): "check_value": check_value, "expect": expect_item, "expect_value": expect_value, + "message": message, } try: - assert_func(check_value, expect_value) + assert_func(check_value, expect_value, message) validate_msg += "\t==> pass" logger.info(validate_msg) validator_dict["check_result"] = "pass" - except AssertionError: + except AssertionError as ex: validate_pass = False validator_dict["check_result"] = "fail" validate_msg += "\t==> fail" @@ -219,6 +237,10 @@ class ResponseObject(object): f"assert_method: {assert_method}\n" f"expect_value: {expect_value}({type(expect_value).__name__})" ) + message = str(ex) + if message: + validate_msg += f"\nmessage: {message}" + logger.error(validate_msg) failures.append(validate_msg) diff --git a/httprunner/testcase.py b/httprunner/testcase.py index 1efb63ec..f6bb0e84 100644 --- a/httprunner/testcase.py +++ b/httprunner/testcase.py @@ -71,140 +71,146 @@ class StepRequestValidation(object): self.__step_context = step_context def assert_equal( - self, jmes_path: Text, expected_value: Any + self, jmes_path: Text, expected_value: Any, message: Text = "" ) -> "StepRequestValidation": - self.__step_context.validators.append({"equal": [jmes_path, expected_value]}) + self.__step_context.validators.append( + {"equal": [jmes_path, expected_value, message]} + ) return self def assert_not_equal( - self, jmes_path: Text, expected_value: Any + self, jmes_path: Text, expected_value: Any, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"not_equal": [jmes_path, expected_value]} + {"not_equal": [jmes_path, expected_value, message]} ) return self def assert_greater_than( - self, jmes_path: Text, expected_value: Union[int, float] + self, jmes_path: Text, expected_value: Union[int, float], message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"greater_than": [jmes_path, expected_value]} + {"greater_than": [jmes_path, expected_value, message]} ) return self def assert_less_than( - self, jmes_path: Text, expected_value: Union[int, float] + self, jmes_path: Text, expected_value: Union[int, float], message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"less_than": [jmes_path, expected_value]} + {"less_than": [jmes_path, expected_value, message]} ) return self def assert_greater_or_equals( - self, jmes_path: Text, expected_value: Union[int, float] + self, jmes_path: Text, expected_value: Union[int, float], message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"greater_or_equals": [jmes_path, expected_value]} + {"greater_or_equals": [jmes_path, expected_value, message]} ) return self def assert_less_or_equals( - self, jmes_path: Text, expected_value: Union[int, float] + self, jmes_path: Text, expected_value: Union[int, float], message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"less_or_equals": [jmes_path, expected_value]} + {"less_or_equals": [jmes_path, expected_value, message]} ) return self def assert_length_equal( - self, jmes_path: Text, expected_value: int + self, jmes_path: Text, expected_value: int, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"length_equal": [jmes_path, expected_value]} + {"length_equal": [jmes_path, expected_value, message]} ) return self def assert_length_greater_than( - self, jmes_path: Text, expected_value: int + self, jmes_path: Text, expected_value: int, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"length_greater_than": [jmes_path, expected_value]} + {"length_greater_than": [jmes_path, expected_value, message]} ) return self def assert_length_less_than( - self, jmes_path: Text, expected_value: int + self, jmes_path: Text, expected_value: int, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"length_less_than": [jmes_path, expected_value]} + {"length_less_than": [jmes_path, expected_value, message]} ) return self def assert_length_greater_or_equals( - self, jmes_path: Text, expected_value: int + self, jmes_path: Text, expected_value: int, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"length_greater_or_equals": [jmes_path, expected_value]} + {"length_greater_or_equals": [jmes_path, expected_value, message]} ) return self def assert_length_less_or_equals( - self, jmes_path: Text, expected_value: int + self, jmes_path: Text, expected_value: int, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"length_less_or_equals": [jmes_path, expected_value]} + {"length_less_or_equals": [jmes_path, expected_value, message]} ) return self def assert_string_equals( - self, jmes_path: Text, expected_value: int + self, jmes_path: Text, expected_value: int, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"string_equals": [jmes_path, expected_value]} + {"string_equals": [jmes_path, expected_value, message]} ) return self def assert_startswith( - self, jmes_path: Text, expected_value: Text + self, jmes_path: Text, expected_value: Text, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"startswith": [jmes_path, expected_value]} + {"startswith": [jmes_path, expected_value, message]} ) return self def assert_endswith( - self, jmes_path: Text, expected_value: Text + self, jmes_path: Text, expected_value: Text, message: Text = "" ) -> "StepRequestValidation": - self.__step_context.validators.append({"endswith": [jmes_path, expected_value]}) + self.__step_context.validators.append( + {"endswith": [jmes_path, expected_value, message]} + ) return self def assert_regex_match( - self, jmes_path: Text, expected_value: Text + self, jmes_path: Text, expected_value: Text, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"regex_match": [jmes_path, expected_value]} + {"regex_match": [jmes_path, expected_value, message]} ) return self def assert_contains( - self, jmes_path: Text, expected_value: Any + self, jmes_path: Text, expected_value: Any, message: Text = "" ) -> "StepRequestValidation": - self.__step_context.validators.append({"contains": [jmes_path, expected_value]}) + self.__step_context.validators.append( + {"contains": [jmes_path, expected_value, message]} + ) return self def assert_contained_by( - self, jmes_path: Text, expected_value: Any + self, jmes_path: Text, expected_value: Any, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"contained_by": [jmes_path, expected_value]} + {"contained_by": [jmes_path, expected_value, message]} ) return self def assert_type_match( - self, jmes_path: Text, expected_value: Text + self, jmes_path: Text, expected_value: Any, message: Text = "" ) -> "StepRequestValidation": self.__step_context.validators.append( - {"type_match": [jmes_path, expected_value]} + {"type_match": [jmes_path, expected_value, message]} ) return self