change validators from dict to list, as there may be several tests on one filed

ate/main.py

@@ -16,7 +16,7 @@ class ApiTestCase(unittest.TestCase):
         """ run testcase and check result.
         """
         result = self.test_runner.run_test(self.testcase)
-        self.assertEqual(result, (True, {}))
+        self.assertEqual(result, (True, []))
 
 def create_suite(testset):
     """ create test suite with a testset, it may include one or several testcases.

ate/response.py

@@ -74,35 +74,36 @@ class ResponseObject(object):
 
     def validate(self, validators, variables_mapping):
         """ Bind named validators to value within the context.
-        @param (dict) validators
-            {
-                "resp_status_code": {"comparator": "eq", "expected": 201},
-                "resp_body_success": {"comparator": "eq", "expected": True}
-            }
+        @param (list) validators
+            [
+                {"check": "status_code", "comparator": "eq", "expected": 201},
+                {"check": "resp_body_success", "comparator": "eq", "expected": True}
+            ]
         @param (dict) variables_mapping
             {
-                "resp_status_code": 200,
                 "resp_body_success": True
             }
-        @return (dict) content differences
-            {
-                "resp_status_code": {
+        @return (list) content differences
+            [
+                {
+                    "check": "status_code",
                     "comparator": "eq", "expected": 201, "value": 200
                 }
-            }
+            ]
         """
-        diff_content_dict = {}
+        diff_content_list = []
 
-        for validator_key, validator_dict in validators.items():
+        for validator_dict in validators:
 
+            if "expected" not in validator_dict or "check" not in validator_dict:
+                raise exception.ParamsError("expected not specified in validator")
+
+            validator_key = validator_dict["check"]
             try:
                 validator_dict["value"] = variables_mapping[validator_key]
             except KeyError:
                 validator_dict["value"] = self.extract_field(validator_key)
 
-            if "expected" not in validator_dict:
-                raise exception.ParamsError("expected not specified in validator")
-
             match_expected = utils.match_expected(
                 validator_dict["value"],
                 validator_dict["expected"],
@@ -110,7 +111,7 @@ class ResponseObject(object):
             )
 
             if not match_expected:
-                diff_content_dict[validator_key] = validator_dict
+                diff_content_list.append(validator_dict)
 
-        self.success = False if diff_content_dict else True
-        return diff_content_dict
+        self.success = False if diff_content_list else True
+        return diff_content_list

ate/runner.py

@@ -59,10 +59,10 @@ class TestRunner(object):
                     "body": '{"name": "user", "password": "123456"}'
                 },
                 "extract_binds": {},
-                "validators": {}
+                "validators": []
             }
         @return (tuple) test result of single testcase
-            (success, diff_content)
+            (success, diff_content_list)
         """
         self.update_context(testcase)
         parsed_request = parse_template(testcase["request"], self.context.variables)
@@ -80,10 +80,10 @@ class TestRunner(object):
         extracted_variables_mapping = resp_obj.extract_response(extract_binds)
         self.context.update_variables(extracted_variables_mapping)
 
-        validators = testcase.get("validators", {})
-        diff_content_dict = resp_obj.validate(validators, self.context.variables)
+        validators = testcase.get("validators", [])
+        diff_content_list = resp_obj.validate(validators, self.context.variables)
 
-        return resp_obj.success, diff_content_dict
+        return resp_obj.success, diff_content_list
 
     def run_testset(self, testset):
         """ run single testset, including one or several testcases.

test/data/demo_template_separate.yml

@@ -22,8 +22,8 @@
             random: "${random}"
         data: "${data}"
     validators:
-        status_code: {"comparator": "eq", "expected": 201}
-        content.success: {"comparator": "eq", "expected": true}
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
 
 - test:
     name: create user which does not exist
@@ -48,5 +48,5 @@
             random: "${random}"
         data: "${data}"
     validators:
-        status_code: {"comparator": "eq", "expected": 500}
-        content.success: {"comparator": "eq", "expected": false}
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

test/data/demo_template_sets.yml

@@ -26,8 +26,8 @@
             random: "${random}"
         data: "${data}"
     validators:
-        status_code: {"comparator": "eq", "expected": 201}
-        content.success: {"comparator": "eq", "expected": true}
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
 
 - test:
     name: create user which does not exist
@@ -43,5 +43,5 @@
             random: "${random}"
         data: "${data}"
     validators:
-        status_code: {"comparator": "eq", "expected": 500}
-        content.success: {"comparator": "eq", "expected": false}
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

test/data/simple_demo_auth_hardcode.json

@@ -15,10 +15,10 @@
                     "password": "123456"
                 }
             },
-            "validators": {
-                "status_code": {"comparator": "eq", "expected": 201},
-                "content.success": {"comparator": "eq", "expected": true}
-            }
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 201},
+                {"check": "content.success", "comparator": "eq", "expected": true}
+            ]
         }
     },
     {
@@ -37,10 +37,10 @@
                     "password": "123456"
                 }
             },
-            "validators": {
-                "status_code": {"comparator": "eq", "expected": 500},
-                "content.success": {"comparator": "eq", "expected": false}
-            }
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 500},
+                {"check": "content.success", "comparator": "eq", "expected": false}
+            ]
         }
     }
 ]

test/data/simple_demo_auth_hardcode.yml

@@ -11,8 +11,8 @@
             name: "user1"
             password: "123456"
     validators:
-        status_code: {"comparator": "eq", "expected": 201}
-        content.success: {"comparator": "eq", "expected": true}
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
 
 - test:
     name: create user which existed
@@ -27,5 +27,5 @@
             name: "user1"
             password: "123456"
     validators:
-        status_code: {"comparator": "eq", "expected": 500}
-        content.success: {"comparator": "eq", "expected": false}
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

test/data/simple_demo_no_auth.json

@@ -14,10 +14,10 @@
                     "password": "123456"
                 }
             },
-            "validators": {
-                "status_code": {"comparator": "eq", "expected": 201},
-                "content.success": {"comparator": "eq", "expected": true}
-            }
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 201},
+                {"check": "content.success", "comparator": "eq", "expected": true}
+            ]
         }
     },
     {
@@ -34,10 +34,10 @@
                     "password": "123456"
                 }
             },
-            "validators": {
-                "status_code": {"comparator": "eq", "expected": 500},
-                "content.success": {"comparator": "eq", "expected": false}
-            }
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 500},
+                {"check": "content.success", "comparator": "eq", "expected": false}
+            ]
         }
     }
 ]

test/data/simple_demo_no_auth.yml

@@ -9,9 +9,9 @@
             name: user1
             password: 123456
     validators:
-        status_code: {"comparator": "eq", "expected": 201}
-        headers.content-type: {"comparator": "eq", "expected": "application/json"}
-        content.success: {"comparator": "eq", "expected": true}
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
+        - {"check": "headers.content-type", "comparator": "eq", "expected": "application/json"}
 
 - test:
     name: create user which existed
@@ -24,6 +24,6 @@
             name: user1
             password: 123456
     validators:
-        status_code: {"comparator": "eq", "expected": 500}
-        headers.content-type: {"comparator": "eq", "expected": "application/json"}
-        content.success: {"comparator": "eq", "expected": false}
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}
+        - {"check": "headers.content-type", "comparator": "eq", "expected": "application/json"}

test/test_response.py

@@ -150,38 +150,39 @@ class TestResponse(ApiServerUnittest):
         resp = requests.get(url)
         resp_obj = response.ResponseObject(resp)
 
-        validators = {
-            "resp_status_code": {"comparator": "eq", "expected": 201},
-            "resp_body_success": {"comparator": "eq", "expected": True}
-        }
+        validators = [
+            {"check": "resp_status_code", "comparator": "eq", "expected": 201},
+            {"check": "resp_body_success", "comparator": "eq", "expected": True}
+        ]
         variables_mapping = {
             "resp_status_code": 200,
             "resp_body_success": True
         }
 
-        diff_content_dict = resp_obj.validate(validators, variables_mapping)
+        diff_content_list = resp_obj.validate(validators, variables_mapping)
         self.assertFalse(resp_obj.success)
         self.assertEqual(
-            diff_content_dict,
-            {
-                "resp_status_code": {
+            diff_content_list,
+            [
+                {
+                    "check": "resp_status_code",
                     "comparator": "eq", "expected": 201, "value": 200
                 }
-            }
+            ]
         )
 
-        validators = {
-            "resp_status_code": {"comparator": "eq", "expected": 201},
-            "resp_body_success": {"comparator": "eq", "expected": True}
-        }
+        validators = [
+            {"check": "resp_status_code", "comparator": "eq", "expected": 201},
+            {"check": "resp_body_success", "comparator": "eq", "expected": True}
+        ]
         variables_mapping = {
             "resp_status_code": 201,
             "resp_body_success": True
         }
 
-        diff_content_dict = resp_obj.validate(validators, variables_mapping)
+        diff_content_list = resp_obj.validate(validators, variables_mapping)
         self.assertTrue(resp_obj.success)
-        self.assertEqual(diff_content_dict, {})
+        self.assertEqual(diff_content_list, [])
 
     def test_validate_exception(self):
         url = "http://127.0.0.1:5000/"
@@ -189,22 +190,19 @@ class TestResponse(ApiServerUnittest):
         resp_obj = response.ResponseObject(resp)
 
         # expected value missed in validators
-        validators = {
-            "resp_status_code": {"comparator": "eq", "expected": 201},
-            "resp_body_success": {"comparator": "eq"}
-        }
-        variables_mapping = {
-            "resp_status_code": 200,
-            "resp_body_success": True
-        }
+        validators = [
+            {"check": "status_code", "comparator": "eq", "expected": 201},
+            {"check": "body_success", "comparator": "eq"}
+        ]
+        variables_mapping = {}
         with self.assertRaises(exception.ParamsError):
             resp_obj.validate(validators, variables_mapping)
 
-        # expected value missed in validators
-        validators = {
-            "resp_status_code": {"comparator": "eq", "expected": 201},
-            "resp_body_success": {"comparator": "eq", "expected": True}
-        }
+        # expected value missed in variables mapping
+        validators = [
+            {"check": "resp_status_code", "comparator": "eq", "expected": 201},
+            {"check": "body_success", "comparator": "eq"}
+        ]
         variables_mapping = {
             "resp_status_code": 200
         }

test/test_runner.py

@@ -46,26 +46,26 @@ class TestRunner(ApiServerUnittest):
                 "resp_body_success": "content.success",
                 "resp_headers_contenttype": "headers.content-type"
             },
-            "validators": {
-                "resp_status_code": {"comparator": "eq", "expected": 200},
-                "resp_body_success": {"comparator": "eq", "expected": False},
-                "resp_headers_contenttype": {"comparator": "eq", "expected": "html/text"}
-            }
+            "validators": [
+                {"check": "resp_status_code", "comparator": "eq", "expected": 200},
+                {"check": "resp_body_success", "comparator": "eq", "expected": False},
+                {"check": "resp_headers_contenttype", "comparator": "eq", "expected": "html/text"}
+            ]
         }
 
-        success, diff_content = self.test_runner.run_test(testcase)
+        success, diff_content_list = self.test_runner.run_test(testcase)
         self.assertFalse(success)
         self.assertEqual(
-            diff_content['resp_status_code'],
-            {"comparator": "eq", "expected": 200, 'value': 201}
+            diff_content_list[0],
+            {"check": "resp_status_code", "comparator": "eq", "expected": 200, 'value': 201}
         )
         self.assertEqual(
-            diff_content['resp_body_success'],
-            {"comparator": "eq", "expected": False, 'value': True}
+            diff_content_list[1],
+            {"check": "resp_body_success", "comparator": "eq", "expected": False, 'value': True}
         )
         self.assertEqual(
-            diff_content['resp_headers_contenttype'],
-            {"comparator": "eq", "expected": "html/text", 'value': "application/json"}
+            diff_content_list[2],
+            {"check": "resp_headers_contenttype", "comparator": "eq", "expected": "html/text", 'value': "application/json"}
         )
 
     def test_run_testset_json_success(self):
@@ -73,25 +73,25 @@ class TestRunner(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(results, [(True, []), (True, [])])
 
     def test_run_testsets_json_success(self):
         testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.json')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, {}), (True, {})])
+        self.assertEqual(results[0], [(True, []), (True, [])])
 
     def test_run_testset_yaml_success(self):
         testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(results, [(True, []), (True, [])])
 
     def test_run_testsets_yaml_success(self):
         testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, {}), (True, {})])
+        self.assertEqual(results[0], [(True, []), (True, [])])

test/test_runner_v2.py

@@ -37,7 +37,7 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(results, [(True, []), (True, [])])
 
     def test_run_testsets_auth_yaml(self):
         testcase_file_path = os.path.join(
@@ -45,7 +45,7 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, {}), (True, {})])
+        self.assertEqual(results[0], [(True, []), (True, [])])
 
     def test_run_testset_auth_json(self):
         testcase_file_path = os.path.join(
@@ -53,7 +53,7 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(results, [(True, []), (True, [])])
 
     def test_run_testsets_auth_json(self):
         testcase_file_path = os.path.join(
@@ -61,7 +61,7 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, {}), (True, {})])
+        self.assertEqual(results[0], [(True, []), (True, [])])
 
     def test_run_testcase_template_yaml(self):
         testcase_file_path = os.path.join(
@@ -78,7 +78,7 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(results, [(True, []), (True, [])])
 
     def test_run_testsets_template_yaml(self):
         testcase_file_path = os.path.join(
@@ -86,4 +86,4 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, {}), (True, {})])
+        self.assertEqual(results[0], [(True, []), (True, [])])