change validators from dict to list, as there may be several tests on one filed

test/data/demo_template_separate.yml

@@ -22,8 +22,8 @@
             random: "${random}"
         data: "${data}"
     validators:
-        status_code: {"comparator": "eq", "expected": 201}
-        content.success: {"comparator": "eq", "expected": true}
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
 
 - test:
     name: create user which does not exist
@@ -48,5 +48,5 @@
             random: "${random}"
         data: "${data}"
     validators:
-        status_code: {"comparator": "eq", "expected": 500}
-        content.success: {"comparator": "eq", "expected": false}
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

test/data/demo_template_sets.yml

@@ -26,8 +26,8 @@
             random: "${random}"
         data: "${data}"
     validators:
-        status_code: {"comparator": "eq", "expected": 201}
-        content.success: {"comparator": "eq", "expected": true}
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
 
 - test:
     name: create user which does not exist
@@ -43,5 +43,5 @@
             random: "${random}"
         data: "${data}"
     validators:
-        status_code: {"comparator": "eq", "expected": 500}
-        content.success: {"comparator": "eq", "expected": false}
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

test/data/simple_demo_auth_hardcode.json

@@ -15,10 +15,10 @@
                     "password": "123456"
                 }
             },
-            "validators": {
-                "status_code": {"comparator": "eq", "expected": 201},
-                "content.success": {"comparator": "eq", "expected": true}
-            }
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 201},
+                {"check": "content.success", "comparator": "eq", "expected": true}
+            ]
         }
     },
     {
@@ -37,10 +37,10 @@
                     "password": "123456"
                 }
             },
-            "validators": {
-                "status_code": {"comparator": "eq", "expected": 500},
-                "content.success": {"comparator": "eq", "expected": false}
-            }
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 500},
+                {"check": "content.success", "comparator": "eq", "expected": false}
+            ]
         }
     }
 ]

test/data/simple_demo_auth_hardcode.yml

@@ -11,8 +11,8 @@
             name: "user1"
             password: "123456"
     validators:
-        status_code: {"comparator": "eq", "expected": 201}
-        content.success: {"comparator": "eq", "expected": true}
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
 
 - test:
     name: create user which existed
@@ -27,5 +27,5 @@
             name: "user1"
             password: "123456"
     validators:
-        status_code: {"comparator": "eq", "expected": 500}
-        content.success: {"comparator": "eq", "expected": false}
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

test/data/simple_demo_no_auth.json

@@ -14,10 +14,10 @@
                     "password": "123456"
                 }
             },
-            "validators": {
-                "status_code": {"comparator": "eq", "expected": 201},
-                "content.success": {"comparator": "eq", "expected": true}
-            }
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 201},
+                {"check": "content.success", "comparator": "eq", "expected": true}
+            ]
         }
     },
     {
@@ -34,10 +34,10 @@
                     "password": "123456"
                 }
             },
-            "validators": {
-                "status_code": {"comparator": "eq", "expected": 500},
-                "content.success": {"comparator": "eq", "expected": false}
-            }
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 500},
+                {"check": "content.success", "comparator": "eq", "expected": false}
+            ]
         }
     }
 ]

test/data/simple_demo_no_auth.yml

@@ -9,9 +9,9 @@
             name: user1
             password: 123456
     validators:
-        status_code: {"comparator": "eq", "expected": 201}
-        headers.content-type: {"comparator": "eq", "expected": "application/json"}
-        content.success: {"comparator": "eq", "expected": true}
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
+        - {"check": "headers.content-type", "comparator": "eq", "expected": "application/json"}
 
 - test:
     name: create user which existed
@@ -24,6 +24,6 @@
             name: user1
             password: 123456
     validators:
-        status_code: {"comparator": "eq", "expected": 500}
-        headers.content-type: {"comparator": "eq", "expected": "application/json"}
-        content.success: {"comparator": "eq", "expected": false}
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}
+        - {"check": "headers.content-type", "comparator": "eq", "expected": "application/json"}

test/test_response.py

@@ -150,38 +150,39 @@ class TestResponse(ApiServerUnittest):
         resp = requests.get(url)
         resp_obj = response.ResponseObject(resp)
 
-        validators = {
-            "resp_status_code": {"comparator": "eq", "expected": 201},
-            "resp_body_success": {"comparator": "eq", "expected": True}
-        }
+        validators = [
+            {"check": "resp_status_code", "comparator": "eq", "expected": 201},
+            {"check": "resp_body_success", "comparator": "eq", "expected": True}
+        ]
         variables_mapping = {
             "resp_status_code": 200,
             "resp_body_success": True
         }
 
-        diff_content_dict = resp_obj.validate(validators, variables_mapping)
+        diff_content_list = resp_obj.validate(validators, variables_mapping)
         self.assertFalse(resp_obj.success)
         self.assertEqual(
-            diff_content_dict,
-            {
-                "resp_status_code": {
+            diff_content_list,
+            [
+                {
+                    "check": "resp_status_code",
                     "comparator": "eq", "expected": 201, "value": 200
                 }
-            }
+            ]
         )
 
-        validators = {
-            "resp_status_code": {"comparator": "eq", "expected": 201},
-            "resp_body_success": {"comparator": "eq", "expected": True}
-        }
+        validators = [
+            {"check": "resp_status_code", "comparator": "eq", "expected": 201},
+            {"check": "resp_body_success", "comparator": "eq", "expected": True}
+        ]
         variables_mapping = {
             "resp_status_code": 201,
             "resp_body_success": True
         }
 
-        diff_content_dict = resp_obj.validate(validators, variables_mapping)
+        diff_content_list = resp_obj.validate(validators, variables_mapping)
         self.assertTrue(resp_obj.success)
-        self.assertEqual(diff_content_dict, {})
+        self.assertEqual(diff_content_list, [])
 
     def test_validate_exception(self):
         url = "http://127.0.0.1:5000/"
@@ -189,22 +190,19 @@ class TestResponse(ApiServerUnittest):
         resp_obj = response.ResponseObject(resp)
 
         # expected value missed in validators
-        validators = {
-            "resp_status_code": {"comparator": "eq", "expected": 201},
-            "resp_body_success": {"comparator": "eq"}
-        }
-        variables_mapping = {
-            "resp_status_code": 200,
-            "resp_body_success": True
-        }
+        validators = [
+            {"check": "status_code", "comparator": "eq", "expected": 201},
+            {"check": "body_success", "comparator": "eq"}
+        ]
+        variables_mapping = {}
         with self.assertRaises(exception.ParamsError):
             resp_obj.validate(validators, variables_mapping)
 
-        # expected value missed in validators
-        validators = {
-            "resp_status_code": {"comparator": "eq", "expected": 201},
-            "resp_body_success": {"comparator": "eq", "expected": True}
-        }
+        # expected value missed in variables mapping
+        validators = [
+            {"check": "resp_status_code", "comparator": "eq", "expected": 201},
+            {"check": "body_success", "comparator": "eq"}
+        ]
         variables_mapping = {
             "resp_status_code": 200
         }

test/test_runner.py

@@ -46,26 +46,26 @@ class TestRunner(ApiServerUnittest):
                 "resp_body_success": "content.success",
                 "resp_headers_contenttype": "headers.content-type"
             },
-            "validators": {
-                "resp_status_code": {"comparator": "eq", "expected": 200},
-                "resp_body_success": {"comparator": "eq", "expected": False},
-                "resp_headers_contenttype": {"comparator": "eq", "expected": "html/text"}
-            }
+            "validators": [
+                {"check": "resp_status_code", "comparator": "eq", "expected": 200},
+                {"check": "resp_body_success", "comparator": "eq", "expected": False},
+                {"check": "resp_headers_contenttype", "comparator": "eq", "expected": "html/text"}
+            ]
         }
 
-        success, diff_content = self.test_runner.run_test(testcase)
+        success, diff_content_list = self.test_runner.run_test(testcase)
         self.assertFalse(success)
         self.assertEqual(
-            diff_content['resp_status_code'],
-            {"comparator": "eq", "expected": 200, 'value': 201}
+            diff_content_list[0],
+            {"check": "resp_status_code", "comparator": "eq", "expected": 200, 'value': 201}
         )
         self.assertEqual(
-            diff_content['resp_body_success'],
-            {"comparator": "eq", "expected": False, 'value': True}
+            diff_content_list[1],
+            {"check": "resp_body_success", "comparator": "eq", "expected": False, 'value': True}
         )
         self.assertEqual(
-            diff_content['resp_headers_contenttype'],
-            {"comparator": "eq", "expected": "html/text", 'value': "application/json"}
+            diff_content_list[2],
+            {"check": "resp_headers_contenttype", "comparator": "eq", "expected": "html/text", 'value': "application/json"}
         )
 
     def test_run_testset_json_success(self):
@@ -73,25 +73,25 @@ class TestRunner(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(results, [(True, []), (True, [])])
 
     def test_run_testsets_json_success(self):
         testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.json')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, {}), (True, {})])
+        self.assertEqual(results[0], [(True, []), (True, [])])
 
     def test_run_testset_yaml_success(self):
         testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(results, [(True, []), (True, [])])
 
     def test_run_testsets_yaml_success(self):
         testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, {}), (True, {})])
+        self.assertEqual(results[0], [(True, []), (True, [])])

test/test_runner_v2.py

@@ -37,7 +37,7 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(results, [(True, []), (True, [])])
 
     def test_run_testsets_auth_yaml(self):
         testcase_file_path = os.path.join(
@@ -45,7 +45,7 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, {}), (True, {})])
+        self.assertEqual(results[0], [(True, []), (True, [])])
 
     def test_run_testset_auth_json(self):
         testcase_file_path = os.path.join(
@@ -53,7 +53,7 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(results, [(True, []), (True, [])])
 
     def test_run_testsets_auth_json(self):
         testcase_file_path = os.path.join(
@@ -61,7 +61,7 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, {}), (True, {})])
+        self.assertEqual(results[0], [(True, []), (True, [])])
 
     def test_run_testcase_template_yaml(self):
         testcase_file_path = os.path.join(
@@ -78,7 +78,7 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(results, [(True, []), (True, [])])
 
     def test_run_testsets_template_yaml(self):
         testcase_file_path = os.path.join(
@@ -86,4 +86,4 @@ class TestRunnerV2(ApiServerUnittest):
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, {}), (True, {})])
+        self.assertEqual(results[0], [(True, []), (True, [])])