diff --git a/ate/response.py b/ate/response.py index 38570388..34dc0f43 100644 --- a/ate/response.py +++ b/ate/response.py @@ -8,7 +8,6 @@ class ResponseObject(object): @param (requests.Response instance) resp_obj """ self.resp_obj = resp_obj - self.success = True def parsed_body(self): try: @@ -95,8 +94,6 @@ class ResponseObject(object): } ] """ - diff_content_list = [] - for validator_dict in validators: check_item = validator_dict.get("check") @@ -117,14 +114,11 @@ class ResponseObject(object): except exception.ParseResponseError: raise exception.ParseResponseError("failed to extract check item in response!") - match_expected = utils.match_expected( + utils.match_expected( validator_dict["actual_value"], expected, - comparator + comparator, + check_item ) - if not match_expected: - diff_content_list.append(validator_dict) - - self.success = False if diff_content_list else True - return diff_content_list + return True diff --git a/ate/runner.py b/ate/runner.py index cf11de8f..0d384a7a 100644 --- a/ate/runner.py +++ b/ate/runner.py @@ -101,10 +101,9 @@ class Runner(object): extracted_variables_mapping_list = resp_obj.extract_response(extract_binds) self.context.bind_variables(extracted_variables_mapping_list, level="testset") - diff_content_list = resp_obj.validate( - validators, self.context.get_testcase_variables_mapping()) + resp_obj.validate(validators, self.context.get_testcase_variables_mapping()) - return resp_obj.success, diff_content_list + return True def run_testset(self, testset): """ run single testset, including one or several testcases. diff --git a/ate/task.py b/ate/task.py index 83dfc6f8..a85f08b9 100644 --- a/ate/task.py +++ b/ate/task.py @@ -14,8 +14,7 @@ class ApiTestCase(unittest.TestCase): def runTest(self): """ run testcase and check result. """ - result = self.test_runner.run_test(self.testcase) - self.assertEqual(result, (True, [])) + self.assertTrue(self.test_runner.run_test(self.testcase)) def create_suite(testset): """ create test suite with a testset, it may include one or several testcases. diff --git a/ate/utils.py b/ate/utils.py index 4127dbaf..432b8a6e 100644 --- a/ate/utils.py +++ b/ate/utils.py @@ -144,11 +144,12 @@ def query_json(json_content, query, delimiter='.'): return json_content -def match_expected(value, expected, comparator="eq"): +def match_expected(value, expected, comparator="eq", check_item=""): """ check if value matches expected value. - @param value: value that get from response. + @param value: actual value that get from response. @param expected: expected result described in testcase @param comparator: compare method + @param check_item: check item name """ try: if comparator in ["eq", "equals", "=="]: @@ -192,7 +193,13 @@ def match_expected(value, expected, comparator="eq"): return True except AssertionError: - return False + err_msg = "\n".join([ + "check item name: %s;" % check_item, + "check item value: %s (%s);" % (value, type(value).__name__), + "comparator: %s;" % comparator, + "expected value: %s (%s)." % (expected, type(expected).__name__) + ]) + raise exception.ValidationError(err_msg) def deep_update_dict(origin_dict, override_dict): """ update origin dict with override dict recursively diff --git a/tests/test_response.py b/tests/test_response.py index 439f8420..b290c947 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -163,17 +163,8 @@ class TestResponse(ApiServerUnittest): "resp_body_success": True } - diff_content_list = resp_obj.validate(validators, variables_mapping) - self.assertFalse(resp_obj.success) - self.assertEqual( - diff_content_list, - [ - { - "check": "resp_status_code", - "comparator": "eq", "expected": 201, "actual_value": 200 - } - ] - ) + with self.assertRaises(exception.ValidationError): + resp_obj.validate(validators, variables_mapping) validators = [ {"check": "resp_status_code", "comparator": "eq", "expected": 201}, @@ -184,9 +175,7 @@ class TestResponse(ApiServerUnittest): "resp_body_success": True } - diff_content_list = resp_obj.validate(validators, variables_mapping) - self.assertTrue(resp_obj.success) - self.assertEqual(diff_content_list, []) + self.assertTrue(resp_obj.validate(validators, variables_mapping)) def test_validate_exception(self): url = "http://127.0.0.1:5000/" @@ -199,7 +188,7 @@ class TestResponse(ApiServerUnittest): {"check": "body_success", "comparator": "eq"} ] variables_mapping = {} - with self.assertRaises(exception.ParamsError): + with self.assertRaises(exception.ValidationError): resp_obj.validate(validators, variables_mapping) # expected value missed in variables mapping @@ -210,5 +199,5 @@ class TestResponse(ApiServerUnittest): variables_mapping = { "resp_status_code": 200 } - with self.assertRaises(exception.ParamsError): + with self.assertRaises(exception.ValidationError): resp_obj.validate(validators, variables_mapping) diff --git a/tests/test_runner.py b/tests/test_runner.py index ace76de8..c28872c0 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -25,16 +25,13 @@ class TestRunner(ApiServerUnittest): for testcase_file_path in self.testcase_file_path_list: testcases = utils.load_testcases(testcase_file_path) testcase = testcases[0]["test"] - success, _ = self.test_runner.run_test(testcase) - self.assertTrue(success) + self.assertTrue(self.test_runner.run_test(testcase)) testcase = testcases[1]["test"] - success, _ = self.test_runner.run_test(testcase) - self.assertTrue(success) + self.assertTrue(self.test_runner.run_test(testcase)) testcase = testcases[2]["test"] - success, _ = self.test_runner.run_test(testcase) - self.assertTrue(success) + self.assertTrue(self.test_runner.run_test(testcase)) def test_run_single_testcase_fail(self): testcase = { @@ -62,26 +59,22 @@ class TestRunner(ApiServerUnittest): ] } - success, diff_content_list = self.test_runner.run_test(testcase) - self.assertFalse(success) - self.assertEqual( - diff_content_list[0], - {"check": "status_code", "comparator": "eq", "expected": 205, 'actual_value': 200} - ) + with self.assertRaises(exception.ValidationError): + self.test_runner.run_test(testcase) def test_run_testset_hardcode(self): for testcase_file_path in self.testcase_file_path_list: testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testset(testsets[0]) self.assertEqual(len(results), 3) - self.assertEqual(results, [(True, [])] * 3) + self.assertEqual(results, [True] * 3) def test_run_testsets_hardcode(self): for testcase_file_path in self.testcase_file_path_list: testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) self.assertEqual(len(results), 1) - self.assertEqual(results, [[(True, [])] * 3]) + self.assertEqual(results, [[True] * 3]) def test_run_testset_template_variables(self): testcase_file_path = os.path.join( @@ -89,7 +82,7 @@ class TestRunner(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testset(testsets[0]) self.assertEqual(len(results), 3) - self.assertEqual(results, [(True, [])] * 3) + self.assertEqual(results, [True] * 3) def test_run_testset_template_import_functions(self): testcase_file_path = os.path.join( @@ -97,7 +90,7 @@ class TestRunner(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testset(testsets[0]) self.assertEqual(len(results), 3) - self.assertEqual(results, [(True, [])] * 3) + self.assertEqual(results, [True] * 3) def test_run_testsets_template_import_functions(self): testcase_file_path = os.path.join( @@ -105,7 +98,7 @@ class TestRunner(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) self.assertEqual(len(results), 1) - self.assertEqual(results, [[(True, [])] * 3]) + self.assertEqual(results, [[True] * 3]) def test_run_testsets_template_lambda_functions(self): testcase_file_path = os.path.join( @@ -113,4 +106,4 @@ class TestRunner(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) self.assertEqual(len(results), 1) - self.assertEqual(results, [[(True, [])] * 3]) + self.assertEqual(results, [[True] * 3]) diff --git a/tests/test_utils.py b/tests/test_utils.py index 863db016..ba01636d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -158,8 +158,12 @@ class TestUtils(ApiServerUnittest): self.assertTrue(utils.match_expected(1, 1, "eq")) self.assertTrue(utils.match_expected("abc", "abc", "eq")) self.assertTrue(utils.match_expected("abc", "abc")) - self.assertFalse(utils.match_expected(123, "123", "eq")) - self.assertFalse(utils.match_expected(123, "123")) + + with self.assertRaises(exception.ValidationError): + utils.match_expected(123, "123", "eq") + + with self.assertRaises(exception.ValidationError): + utils.match_expected(123, "123") self.assertTrue(utils.match_expected("123", 3, "len_eq")) self.assertTrue(utils.match_expected(123, "123", "str_eq")) @@ -179,7 +183,8 @@ class TestUtils(ApiServerUnittest): self.assertTrue(utils.match_expected("3ab", "123abc456", "contained_by")) self.assertTrue(utils.match_expected("123abc456", "^123.*456$", "regex")) - self.assertFalse(utils.match_expected("123abc456", "^12b.*456$", "regex")) + with self.assertRaises(exception.ValidationError): + utils.match_expected("123abc456", "^12b.*456$", "regex") with self.assertRaises(exception.ParamsError): utils.match_expected(1, 2, "not_supported_comparator")