optimize response validation: if validation failed, then raise ValidationError directly

ate/response.py

@@ -8,7 +8,6 @@ class ResponseObject(object):
         @param (requests.Response instance) resp_obj
         """
         self.resp_obj = resp_obj
-        self.success = True
 
     def parsed_body(self):
         try:
@@ -95,8 +94,6 @@ class ResponseObject(object):
                 }
             ]
         """
-        diff_content_list = []
-
         for validator_dict in validators:
 
             check_item = validator_dict.get("check")
@@ -117,14 +114,11 @@ class ResponseObject(object):
                 except exception.ParseResponseError:
                     raise exception.ParseResponseError("failed to extract check item in response!")
 
-            match_expected = utils.match_expected(
+            utils.match_expected(
                 validator_dict["actual_value"],
                 expected,
-                comparator
+                comparator,
+                check_item
             )
 
-            if not match_expected:
-                diff_content_list.append(validator_dict)
-
-        self.success = False if diff_content_list else True
-        return diff_content_list
+        return True

ate/runner.py

@@ -101,10 +101,9 @@ class Runner(object):
             extracted_variables_mapping_list = resp_obj.extract_response(extract_binds)
             self.context.bind_variables(extracted_variables_mapping_list, level="testset")
 
-            diff_content_list = resp_obj.validate(
-                validators, self.context.get_testcase_variables_mapping())
+            resp_obj.validate(validators, self.context.get_testcase_variables_mapping())
 
-        return resp_obj.success, diff_content_list
+        return True
 
     def run_testset(self, testset):
         """ run single testset, including one or several testcases.

ate/task.py

@@ -14,8 +14,7 @@ class ApiTestCase(unittest.TestCase):
     def runTest(self):
         """ run testcase and check result.
         """
-        result = self.test_runner.run_test(self.testcase)
-        self.assertEqual(result, (True, []))
+        self.assertTrue(self.test_runner.run_test(self.testcase))
 
 def create_suite(testset):
     """ create test suite with a testset, it may include one or several testcases.

ate/utils.py

@@ -144,11 +144,12 @@ def query_json(json_content, query, delimiter='.'):
 
     return json_content
 
-def match_expected(value, expected, comparator="eq"):
+def match_expected(value, expected, comparator="eq", check_item=""):
     """ check if value matches expected value.
-    @param value: value that get from response.
+    @param value: actual value that get from response.
     @param expected: expected result described in testcase
     @param comparator: compare method
+    @param check_item: check item name
     """
     try:
         if comparator in ["eq", "equals", "=="]:
@@ -192,7 +193,13 @@ def match_expected(value, expected, comparator="eq"):
 
         return True
     except AssertionError:
-        return False
+        err_msg = "\n".join([
+            "check item name: %s;" % check_item,
+            "check item value: %s (%s);" % (value, type(value).__name__),
+            "comparator: %s;" % comparator,
+            "expected value: %s (%s)." % (expected, type(expected).__name__)
+        ])
+        raise exception.ValidationError(err_msg)
 
 def deep_update_dict(origin_dict, override_dict):
     """ update origin dict with override dict recursively

tests/test_response.py

@@ -163,17 +163,8 @@ class TestResponse(ApiServerUnittest):
             "resp_body_success": True
         }
 
-        diff_content_list = resp_obj.validate(validators, variables_mapping)
-        self.assertFalse(resp_obj.success)
-        self.assertEqual(
-            diff_content_list,
-            [
-                {
-                    "check": "resp_status_code",
-                    "comparator": "eq", "expected": 201, "actual_value": 200
-                }
-            ]
-        )
+        with self.assertRaises(exception.ValidationError):
+            resp_obj.validate(validators, variables_mapping)
 
         validators = [
             {"check": "resp_status_code", "comparator": "eq", "expected": 201},
@@ -184,9 +175,7 @@ class TestResponse(ApiServerUnittest):
             "resp_body_success": True
         }
 
-        diff_content_list = resp_obj.validate(validators, variables_mapping)
-        self.assertTrue(resp_obj.success)
-        self.assertEqual(diff_content_list, [])
+        self.assertTrue(resp_obj.validate(validators, variables_mapping))
 
     def test_validate_exception(self):
         url = "http://127.0.0.1:5000/"
@@ -199,7 +188,7 @@ class TestResponse(ApiServerUnittest):
             {"check": "body_success", "comparator": "eq"}
         ]
         variables_mapping = {}
-        with self.assertRaises(exception.ParamsError):
+        with self.assertRaises(exception.ValidationError):
             resp_obj.validate(validators, variables_mapping)
 
         # expected value missed in variables mapping
@@ -210,5 +199,5 @@ class TestResponse(ApiServerUnittest):
         variables_mapping = {
             "resp_status_code": 200
         }
-        with self.assertRaises(exception.ParamsError):
+        with self.assertRaises(exception.ValidationError):
             resp_obj.validate(validators, variables_mapping)

tests/test_runner.py

@@ -25,16 +25,13 @@ class TestRunner(ApiServerUnittest):
         for testcase_file_path in self.testcase_file_path_list:
             testcases = utils.load_testcases(testcase_file_path)
             testcase = testcases[0]["test"]
-            success, _ = self.test_runner.run_test(testcase)
-            self.assertTrue(success)
+            self.assertTrue(self.test_runner.run_test(testcase))
 
             testcase = testcases[1]["test"]
-            success, _ = self.test_runner.run_test(testcase)
-            self.assertTrue(success)
+            self.assertTrue(self.test_runner.run_test(testcase))
 
             testcase = testcases[2]["test"]
-            success, _ = self.test_runner.run_test(testcase)
-            self.assertTrue(success)
+            self.assertTrue(self.test_runner.run_test(testcase))
 
     def test_run_single_testcase_fail(self):
         testcase = {
@@ -62,26 +59,22 @@ class TestRunner(ApiServerUnittest):
             ]
         }
 
-        success, diff_content_list = self.test_runner.run_test(testcase)
-        self.assertFalse(success)
-        self.assertEqual(
-            diff_content_list[0],
-            {"check": "status_code", "comparator": "eq", "expected": 205, 'actual_value': 200}
-        )
+        with self.assertRaises(exception.ValidationError):
+            self.test_runner.run_test(testcase)
 
     def test_run_testset_hardcode(self):
         for testcase_file_path in self.testcase_file_path_list:
             testsets = utils.load_testcases_by_path(testcase_file_path)
             results = self.test_runner.run_testset(testsets[0])
             self.assertEqual(len(results), 3)
-            self.assertEqual(results, [(True, [])] * 3)
+            self.assertEqual(results, [True] * 3)
 
     def test_run_testsets_hardcode(self):
         for testcase_file_path in self.testcase_file_path_list:
             testsets = utils.load_testcases_by_path(testcase_file_path)
             results = self.test_runner.run_testsets(testsets)
             self.assertEqual(len(results), 1)
-            self.assertEqual(results, [[(True, [])] * 3])
+            self.assertEqual(results, [[True] * 3])
 
     def test_run_testset_template_variables(self):
         testcase_file_path = os.path.join(
@@ -89,7 +82,7 @@ class TestRunner(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 3)
-        self.assertEqual(results, [(True, [])] * 3)
+        self.assertEqual(results, [True] * 3)
 
     def test_run_testset_template_import_functions(self):
         testcase_file_path = os.path.join(
@@ -97,7 +90,7 @@ class TestRunner(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 3)
-        self.assertEqual(results, [(True, [])] * 3)
+        self.assertEqual(results, [True] * 3)
 
     def test_run_testsets_template_import_functions(self):
         testcase_file_path = os.path.join(
@@ -105,7 +98,7 @@ class TestRunner(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results, [[(True, [])] * 3])
+        self.assertEqual(results, [[True] * 3])
 
     def test_run_testsets_template_lambda_functions(self):
         testcase_file_path = os.path.join(
@@ -113,4 +106,4 @@ class TestRunner(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results, [[(True, [])] * 3])
+        self.assertEqual(results, [[True] * 3])

tests/test_utils.py

@@ -158,8 +158,12 @@ class TestUtils(ApiServerUnittest):
         self.assertTrue(utils.match_expected(1, 1, "eq"))
         self.assertTrue(utils.match_expected("abc", "abc", "eq"))
         self.assertTrue(utils.match_expected("abc", "abc"))
-        self.assertFalse(utils.match_expected(123, "123", "eq"))
-        self.assertFalse(utils.match_expected(123, "123"))
+
+        with self.assertRaises(exception.ValidationError):
+            utils.match_expected(123, "123", "eq")
+
+        with self.assertRaises(exception.ValidationError):
+            utils.match_expected(123, "123")
 
         self.assertTrue(utils.match_expected("123", 3, "len_eq"))
         self.assertTrue(utils.match_expected(123, "123", "str_eq"))
@@ -179,7 +183,8 @@ class TestUtils(ApiServerUnittest):
         self.assertTrue(utils.match_expected("3ab", "123abc456", "contained_by"))
 
         self.assertTrue(utils.match_expected("123abc456", "^123.*456$", "regex"))
-        self.assertFalse(utils.match_expected("123abc456", "^12b.*456$", "regex"))
+        with self.assertRaises(exception.ValidationError):
+            utils.match_expected("123abc456", "^12b.*456$", "regex")
 
         with self.assertRaises(exception.ParamsError):
             utils.match_expected(1, 2, "not_supported_comparator")