diff --git a/httprunner/context.py b/httprunner/context.py index ac9b5b3c..967f36a2 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -169,54 +169,22 @@ class Context(object): """ self.testcase_parser.eval_content_functions(content) - def parse_validator(self, validator, resp_obj): - """ parse validator, validator maybe in two format + def eval_check_item(self, validator, resp_obj): + """ evaluate check item in validator @param (dict) validator - format1: this is kept for compatiblity with the previous versions. - {"check": "status_code", "comparator": "eq", "expect": 201} - {"check": "$resp_body_success", "comparator": "eq", "expect": True} - format2: recommended new version - {'eq': ['status_code', 201]} - {'eq': ['$resp_body_success', True]} + {"check": "status_code", "comparator": "eq", "expect": 201} + {"check": "$resp_body_success", "comparator": "eq", "expect": True} @param (object) resp_obj @return (dict) validator info { - "check_item": check_item, - "check_value": check_value, - "expect_value": expect_value, - "comparator": comparator + "check": "status_code", + "check_value": 200, + "expect": 201, + "comparator": "eq" } """ - if not isinstance(validator, dict): - raise exception.ParamsError("invalid validator: {}".format(validator)) - - if "check" in validator and len(validator) > 1: - # format1 - check_item = validator.get("check") - - if "expect" in validator: - expect_value = validator.get("expect") - elif "expected" in validator: - expect_value = validator.get("expected") - else: - raise exception.ParamsError("invalid validator: {}".format(validator)) - - comparator = validator.get("comparator", "eq") - - elif len(validator) == 1: - # format2 - comparator = list(validator.keys())[0] - compare_values = validator[comparator] - - if not isinstance(compare_values, list) or len(compare_values) != 2: - raise exception.ParamsError("invalid validator: {}".format(validator)) - - check_item, expect_value = compare_values - - else: - raise exception.ParamsError("invalid validator: {}".format(validator)) - - # check_item should only be in 3 type: + check_item = validator["check"] + # check_item should only be in 3 types: # 1, variable reference, e.g. $token # 2, string joined by delimiter. e.g. "status_code", "headers.content-type" # 3, regex string, e.g. "LB[\d]*(.*)RB[\d]*" @@ -230,15 +198,14 @@ class Context(object): except exception.ParseResponseError: raise exception.ParseResponseError("failed to extract check item in response!") - expect_value = self.testcase_parser.eval_content_variables(expect_value) + validator["check_value"] = check_value - validator_dict = { - "check_item": check_item, - "check_value": check_value, - "expect_value": expect_value, - "comparator": comparator - } - return validator_dict + # expect_value should only be in 2 types: + # 1, variable reference, e.g. $expect_status_code + # 2, actual value, e.g. 200 + expect_value = self.testcase_parser.eval_content_variables(validator["expect"]) + validator["expect"] = expect_value + return validator def do_validation(self, validator_dict): """ validate with functions @@ -249,15 +216,15 @@ class Context(object): if not validate_func: raise exception.FunctionNotFound("comparator not found: {}".format(comparator)) - check_item = validator_dict["check_item"] + check_item = validator_dict["check"] check_value = validator_dict["check_value"] - expect_value = validator_dict["expect_value"] + expect_value = validator_dict["expect"] try: if check_value is None or expect_value is None: assert comparator in ["is", "eq", "equals", "=="] - validate_func(validator_dict["check_value"], validator_dict["expect_value"]) + validate_func(validator_dict["check_value"], validator_dict["expect"]) except (AssertionError, TypeError): err_msg = "\n" + "\n".join([ "\tcheck item name: %s;" % check_item, @@ -273,7 +240,10 @@ class Context(object): @param (object) resp_obj """ for validator in validators: - validator_dict = self.parse_validator(validator, resp_obj) + validator_dict = self.eval_check_item( + testcase.parse_validator(validator), + resp_obj + ) self.do_validation(validator_dict) return True diff --git a/httprunner/runner.py b/httprunner/runner.py index 603da4e3..26a05095 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -105,8 +105,7 @@ class Runner(object): extractors = testcase_dict.get("extract") \ or testcase_dict.get("extractors") \ or testcase_dict.get("extract_binds", []) - validators = testcase_dict.get("validate") \ - or testcase_dict.get("validators", []) + validators = testcase_dict.get("validate", []) setup_actions = testcase_dict.get("setup", []) teardown_actions = testcase_dict.get("teardown", []) diff --git a/httprunner/testcase.py b/httprunner/testcase.py index c83204ac..e1c221e4 100644 --- a/httprunner/testcase.py +++ b/httprunner/testcase.py @@ -212,6 +212,121 @@ def load_testcases_by_path(path): testcases_cache_mapping[path] = testcases_list return testcases_list +def parse_validator(validator): + """ parse validator, validator maybe in two format + @param (dict) validator + format1: this is kept for compatiblity with the previous versions. + {"check": "status_code", "comparator": "eq", "expect": 201} + {"check": "$resp_body_success", "comparator": "eq", "expect": True} + format2: recommended new version + {'eq': ['status_code', 201]} + {'eq': ['$resp_body_success', True]} + @return (dict) validator info + { + "check": "status_code", + "expect": 201, + "comparator": "eq" + } + """ + if not isinstance(validator, dict): + raise exception.ParamsError("invalid validator: {}".format(validator)) + + if "check" in validator and len(validator) > 1: + # format1 + check_item = validator.get("check") + + if "expect" in validator: + expect_value = validator.get("expect") + elif "expected" in validator: + expect_value = validator.get("expected") + else: + raise exception.ParamsError("invalid validator: {}".format(validator)) + + comparator = validator.get("comparator", "eq") + + elif len(validator) == 1: + # format2 + comparator = list(validator.keys())[0] + compare_values = validator[comparator] + + if not isinstance(compare_values, list) or len(compare_values) != 2: + raise exception.ParamsError("invalid validator: {}".format(validator)) + + check_item, expect_value = compare_values + + else: + raise exception.ParamsError("invalid validator: {}".format(validator)) + + return { + "check": check_item, + "expect": expect_value, + "comparator": comparator + } + +def merge_validator(api_validators, test_validators): + """ merge api_validators with test_validators + @params: + api_validators: [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] + test_validators: [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] + @return: + [ + {"check": "v1", "expect": 201, "comparator": "eq"}, + {"check": "s2", "expect": 16, "comparator": "len_eq"}, + {"check": "s3", "expect": 12, "comparator": "len_eq"} + ] + """ + if not api_validators: + return test_validators + + elif not test_validators: + return api_validators + + else: + api_validators_mapping = {} + for api_validator in api_validators: + api_validator = parse_validator(api_validator) + key = (api_validator["check"], api_validator["comparator"]) + api_validators_mapping[key] = api_validator + + test_validators_mapping = {} + for test_validator in test_validators: + test_validator = parse_validator(test_validator) + key = (test_validator["check"], test_validator["comparator"]) + test_validators_mapping[key] = test_validator + + api_validators_mapping.update(test_validators_mapping) + return list(api_validators_mapping.values()) + +def extend_test_api(test_block_dict): + """ update test block api with api definition + @param + test_block_dict: + { + "name": "get token", + "api": "get_token($user_agent, $device_sn, $os_platform, $app_version)", + "extract": [{"token": "content.token"}], + "validate": [{'eq': ['status_code', 200]}, {'len_eq': ['content.token', 16]}] + } + @return + { + "name": "get token", + "api": "get_token($user_agent, $device_sn, $os_platform, $app_version)", + "extract": [{"token": "content.token"}], + "validate": [{'eq': ['status_code', 200]}, {'len_eq': ['content.token', 16]}] + } + """ + ref_name = test_block_dict["api"] + test_info = get_testinfo_by_reference(ref_name, "api") + + api_validators = test_info.get("validate") or test_info.get("validators", []) + test_validators = test_block_dict.get("validate") or test_block_dict.get("validators", []) + + test_block_dict.update(test_info) + test_block_dict["validate"] = merge_validator( + api_validators, + test_validators + ) + def load_test_file(file_path): """ load testset file, get testset data structure. @param file_path: absolute valid testset file path @@ -242,9 +357,7 @@ def load_test_file(file_path): elif key == "test": test_block_dict = item["test"] if "api" in test_block_dict: - ref_name = test_block_dict["api"] - test_info = get_testinfo_by_reference(ref_name, "api") - test_block_dict.update(test_info) + extend_test_api(test_block_dict) testset["testcases"].append(test_block_dict) elif "suite" in test_block_dict: ref_name = test_block_dict["suite"] diff --git a/tests/api/demo.yml b/tests/api/demo.yml index cd163775..5ae3c733 100644 --- a/tests/api/demo.yml +++ b/tests/api/demo.yml @@ -11,8 +11,8 @@ json: sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)} validate: - - {"check": "status_code", "comparator": "eq", "expect": 200} - - {"check": "content.token", "comparator": "len_eq", "expect": 16} + - "eq": ["status_code", 0] + - "len_eq": ["content.token", 12] - api: def: create_user($uid, $user_name, $user_password, $token) diff --git a/tests/data/demo_testset_layer.yml b/tests/data/demo_testset_layer.yml index 1b70b170..bb2b9cd6 100644 --- a/tests/data/demo_testset_layer.yml +++ b/tests/data/demo_testset_layer.yml @@ -18,6 +18,9 @@ api: get_token($user_agent, $device_sn, $os_platform, $app_version) extract: - token: content.token + validate: + - "eq": ["status_code", 200] + - "len_eq": ["content.token", 16] - test: name: reset all users diff --git a/tests/test_context.py b/tests/test_context.py index b4f88a06..56ae0e4c 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -220,10 +220,10 @@ class VariableBindsUnittest(ApiServerUnittest): def test_do_validation(self): self.context.do_validation( - {"check_item": "check_item", "check_value": 1, "expect_value": 1, "comparator": "eq"} + {"check": "check", "check_value": 1, "expect": 1, "comparator": "eq"} ) self.context.do_validation( - {"check_item": "check_item", "check_value": "abc", "expect_value": "abc", "comparator": "=="} + {"check": "check", "check_value": "abc", "expect": "abc", "comparator": "=="} ) config_dict = { @@ -231,7 +231,7 @@ class VariableBindsUnittest(ApiServerUnittest): } self.context.config_context(config_dict, "testset") self.context.do_validation( - {"check_item": "status_code", "check_value": "201", "expect_value": 3, "comparator": "sum_status_code"} + {"check": "status_code", "check_value": "201", "expect": 3, "comparator": "sum_status_code"} ) def test_validate(self): diff --git a/tests/test_testcase.py b/tests/test_testcase.py index d9359061..e6318727 100644 --- a/tests/test_testcase.py +++ b/tests/test_testcase.py @@ -542,3 +542,40 @@ class TestcaseParserUnittest(unittest.TestCase): with self.assertRaises(ApiNotFound): testcase.get_test_definition("api_not_exist", "api") + + def test_parse_validator(self): + validator = {"check": "status_code", "comparator": "eq", "expect": 201} + self.assertEqual( + testcase.parse_validator(validator), + {"check": "status_code", "comparator": "eq", "expect": 201} + ) + + validator = {'eq': ['status_code', 201]} + self.assertEqual( + testcase.parse_validator(validator), + {"check": "status_code", "comparator": "eq", "expect": 201} + ) + + def test_merge_validator(self): + api_validators = [ + {'eq': ['v1', 200]}, + {"check": "s2", "expect": 16, "comparator": "len_eq"} + ] + test_validators = [ + {"check": "v1", "expect": 201}, + {'len_eq': ['s3', 12]} + ] + + merged_validators = testcase.merge_validator(api_validators, test_validators) + self.assertIn( + {"check": "v1", "expect": 201, "comparator": "eq"}, + merged_validators + ) + self.assertIn( + {"check": "s2", "expect": 16, "comparator": "len_eq"}, + merged_validators + ) + self.assertIn( + {"check": "s3", "expect": 12, "comparator": "len_eq"}, + merged_validators + )