diff --git a/ate/main.py b/ate/main.py index 6e0e3cfb..997b3da1 100644 --- a/ate/main.py +++ b/ate/main.py @@ -16,7 +16,7 @@ class ApiTestCase(unittest.TestCase): """ run testcase and check result. """ result = self.test_runner.run_test(self.testcase) - self.assertEqual(result, (True, {})) + self.assertEqual(result, (True, [])) def create_suite(testset): """ create test suite with a testset, it may include one or several testcases. diff --git a/ate/response.py b/ate/response.py index 15e08f45..a5d73f93 100644 --- a/ate/response.py +++ b/ate/response.py @@ -74,35 +74,36 @@ class ResponseObject(object): def validate(self, validators, variables_mapping): """ Bind named validators to value within the context. - @param (dict) validators - { - "resp_status_code": {"comparator": "eq", "expected": 201}, - "resp_body_success": {"comparator": "eq", "expected": True} - } + @param (list) validators + [ + {"check": "status_code", "comparator": "eq", "expected": 201}, + {"check": "resp_body_success", "comparator": "eq", "expected": True} + ] @param (dict) variables_mapping { - "resp_status_code": 200, "resp_body_success": True } - @return (dict) content differences - { - "resp_status_code": { + @return (list) content differences + [ + { + "check": "status_code", "comparator": "eq", "expected": 201, "value": 200 } - } + ] """ - diff_content_dict = {} + diff_content_list = [] - for validator_key, validator_dict in validators.items(): + for validator_dict in validators: + if "expected" not in validator_dict or "check" not in validator_dict: + raise exception.ParamsError("expected not specified in validator") + + validator_key = validator_dict["check"] try: validator_dict["value"] = variables_mapping[validator_key] except KeyError: validator_dict["value"] = self.extract_field(validator_key) - if "expected" not in validator_dict: - raise exception.ParamsError("expected not specified in validator") - match_expected = utils.match_expected( validator_dict["value"], validator_dict["expected"], @@ -110,7 +111,7 @@ class ResponseObject(object): ) if not match_expected: - diff_content_dict[validator_key] = validator_dict + diff_content_list.append(validator_dict) - self.success = False if diff_content_dict else True - return diff_content_dict + self.success = False if diff_content_list else True + return diff_content_list diff --git a/ate/runner.py b/ate/runner.py index 2c3b4f00..f1119839 100644 --- a/ate/runner.py +++ b/ate/runner.py @@ -59,10 +59,10 @@ class TestRunner(object): "body": '{"name": "user", "password": "123456"}' }, "extract_binds": {}, - "validators": {} + "validators": [] } @return (tuple) test result of single testcase - (success, diff_content) + (success, diff_content_list) """ self.update_context(testcase) parsed_request = parse_template(testcase["request"], self.context.variables) @@ -80,10 +80,10 @@ class TestRunner(object): extracted_variables_mapping = resp_obj.extract_response(extract_binds) self.context.update_variables(extracted_variables_mapping) - validators = testcase.get("validators", {}) - diff_content_dict = resp_obj.validate(validators, self.context.variables) + validators = testcase.get("validators", []) + diff_content_list = resp_obj.validate(validators, self.context.variables) - return resp_obj.success, diff_content_dict + return resp_obj.success, diff_content_list def run_testset(self, testset): """ run single testset, including one or several testcases. diff --git a/test/data/demo_template_separate.yml b/test/data/demo_template_separate.yml index 6a1b351e..389d49b9 100644 --- a/test/data/demo_template_separate.yml +++ b/test/data/demo_template_separate.yml @@ -22,8 +22,8 @@ random: "${random}" data: "${data}" validators: - status_code: {"comparator": "eq", "expected": 201} - content.success: {"comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expected": 201} + - {"check": "content.success", "comparator": "eq", "expected": true} - test: name: create user which does not exist @@ -48,5 +48,5 @@ random: "${random}" data: "${data}" validators: - status_code: {"comparator": "eq", "expected": 500} - content.success: {"comparator": "eq", "expected": false} + - {"check": "status_code", "comparator": "eq", "expected": 500} + - {"check": "content.success", "comparator": "eq", "expected": false} diff --git a/test/data/demo_template_sets.yml b/test/data/demo_template_sets.yml index 754433a4..1036e7bd 100644 --- a/test/data/demo_template_sets.yml +++ b/test/data/demo_template_sets.yml @@ -26,8 +26,8 @@ random: "${random}" data: "${data}" validators: - status_code: {"comparator": "eq", "expected": 201} - content.success: {"comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expected": 201} + - {"check": "content.success", "comparator": "eq", "expected": true} - test: name: create user which does not exist @@ -43,5 +43,5 @@ random: "${random}" data: "${data}" validators: - status_code: {"comparator": "eq", "expected": 500} - content.success: {"comparator": "eq", "expected": false} + - {"check": "status_code", "comparator": "eq", "expected": 500} + - {"check": "content.success", "comparator": "eq", "expected": false} diff --git a/test/data/simple_demo_auth_hardcode.json b/test/data/simple_demo_auth_hardcode.json index 840288dc..213c61b1 100644 --- a/test/data/simple_demo_auth_hardcode.json +++ b/test/data/simple_demo_auth_hardcode.json @@ -15,10 +15,10 @@ "password": "123456" } }, - "validators": { - "status_code": {"comparator": "eq", "expected": 201}, - "content.success": {"comparator": "eq", "expected": true} - } + "validators": [ + {"check": "status_code", "comparator": "eq", "expected": 201}, + {"check": "content.success", "comparator": "eq", "expected": true} + ] } }, { @@ -37,10 +37,10 @@ "password": "123456" } }, - "validators": { - "status_code": {"comparator": "eq", "expected": 500}, - "content.success": {"comparator": "eq", "expected": false} - } + "validators": [ + {"check": "status_code", "comparator": "eq", "expected": 500}, + {"check": "content.success", "comparator": "eq", "expected": false} + ] } } ] \ No newline at end of file diff --git a/test/data/simple_demo_auth_hardcode.yml b/test/data/simple_demo_auth_hardcode.yml index 15f66e06..21f432db 100644 --- a/test/data/simple_demo_auth_hardcode.yml +++ b/test/data/simple_demo_auth_hardcode.yml @@ -11,8 +11,8 @@ name: "user1" password: "123456" validators: - status_code: {"comparator": "eq", "expected": 201} - content.success: {"comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expected": 201} + - {"check": "content.success", "comparator": "eq", "expected": true} - test: name: create user which existed @@ -27,5 +27,5 @@ name: "user1" password: "123456" validators: - status_code: {"comparator": "eq", "expected": 500} - content.success: {"comparator": "eq", "expected": false} \ No newline at end of file + - {"check": "status_code", "comparator": "eq", "expected": 500} + - {"check": "content.success", "comparator": "eq", "expected": false} \ No newline at end of file diff --git a/test/data/simple_demo_no_auth.json b/test/data/simple_demo_no_auth.json index ae6c2aaa..35bb895a 100644 --- a/test/data/simple_demo_no_auth.json +++ b/test/data/simple_demo_no_auth.json @@ -14,10 +14,10 @@ "password": "123456" } }, - "validators": { - "status_code": {"comparator": "eq", "expected": 201}, - "content.success": {"comparator": "eq", "expected": true} - } + "validators": [ + {"check": "status_code", "comparator": "eq", "expected": 201}, + {"check": "content.success", "comparator": "eq", "expected": true} + ] } }, { @@ -34,10 +34,10 @@ "password": "123456" } }, - "validators": { - "status_code": {"comparator": "eq", "expected": 500}, - "content.success": {"comparator": "eq", "expected": false} - } + "validators": [ + {"check": "status_code", "comparator": "eq", "expected": 500}, + {"check": "content.success", "comparator": "eq", "expected": false} + ] } } ] \ No newline at end of file diff --git a/test/data/simple_demo_no_auth.yml b/test/data/simple_demo_no_auth.yml index 7a6bf078..aa633577 100644 --- a/test/data/simple_demo_no_auth.yml +++ b/test/data/simple_demo_no_auth.yml @@ -9,9 +9,9 @@ name: user1 password: 123456 validators: - status_code: {"comparator": "eq", "expected": 201} - headers.content-type: {"comparator": "eq", "expected": "application/json"} - content.success: {"comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expected": 201} + - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "headers.content-type", "comparator": "eq", "expected": "application/json"} - test: name: create user which existed @@ -24,6 +24,6 @@ name: user1 password: 123456 validators: - status_code: {"comparator": "eq", "expected": 500} - headers.content-type: {"comparator": "eq", "expected": "application/json"} - content.success: {"comparator": "eq", "expected": false} \ No newline at end of file + - {"check": "status_code", "comparator": "eq", "expected": 500} + - {"check": "content.success", "comparator": "eq", "expected": false} + - {"check": "headers.content-type", "comparator": "eq", "expected": "application/json"} diff --git a/test/test_response.py b/test/test_response.py index c4d02723..cbeb01d6 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -150,38 +150,39 @@ class TestResponse(ApiServerUnittest): resp = requests.get(url) resp_obj = response.ResponseObject(resp) - validators = { - "resp_status_code": {"comparator": "eq", "expected": 201}, - "resp_body_success": {"comparator": "eq", "expected": True} - } + validators = [ + {"check": "resp_status_code", "comparator": "eq", "expected": 201}, + {"check": "resp_body_success", "comparator": "eq", "expected": True} + ] variables_mapping = { "resp_status_code": 200, "resp_body_success": True } - diff_content_dict = resp_obj.validate(validators, variables_mapping) + diff_content_list = resp_obj.validate(validators, variables_mapping) self.assertFalse(resp_obj.success) self.assertEqual( - diff_content_dict, - { - "resp_status_code": { + diff_content_list, + [ + { + "check": "resp_status_code", "comparator": "eq", "expected": 201, "value": 200 } - } + ] ) - validators = { - "resp_status_code": {"comparator": "eq", "expected": 201}, - "resp_body_success": {"comparator": "eq", "expected": True} - } + validators = [ + {"check": "resp_status_code", "comparator": "eq", "expected": 201}, + {"check": "resp_body_success", "comparator": "eq", "expected": True} + ] variables_mapping = { "resp_status_code": 201, "resp_body_success": True } - diff_content_dict = resp_obj.validate(validators, variables_mapping) + diff_content_list = resp_obj.validate(validators, variables_mapping) self.assertTrue(resp_obj.success) - self.assertEqual(diff_content_dict, {}) + self.assertEqual(diff_content_list, []) def test_validate_exception(self): url = "http://127.0.0.1:5000/" @@ -189,22 +190,19 @@ class TestResponse(ApiServerUnittest): resp_obj = response.ResponseObject(resp) # expected value missed in validators - validators = { - "resp_status_code": {"comparator": "eq", "expected": 201}, - "resp_body_success": {"comparator": "eq"} - } - variables_mapping = { - "resp_status_code": 200, - "resp_body_success": True - } + validators = [ + {"check": "status_code", "comparator": "eq", "expected": 201}, + {"check": "body_success", "comparator": "eq"} + ] + variables_mapping = {} with self.assertRaises(exception.ParamsError): resp_obj.validate(validators, variables_mapping) - # expected value missed in validators - validators = { - "resp_status_code": {"comparator": "eq", "expected": 201}, - "resp_body_success": {"comparator": "eq", "expected": True} - } + # expected value missed in variables mapping + validators = [ + {"check": "resp_status_code", "comparator": "eq", "expected": 201}, + {"check": "body_success", "comparator": "eq"} + ] variables_mapping = { "resp_status_code": 200 } diff --git a/test/test_runner.py b/test/test_runner.py index 43b122a4..729ec460 100644 --- a/test/test_runner.py +++ b/test/test_runner.py @@ -46,26 +46,26 @@ class TestRunner(ApiServerUnittest): "resp_body_success": "content.success", "resp_headers_contenttype": "headers.content-type" }, - "validators": { - "resp_status_code": {"comparator": "eq", "expected": 200}, - "resp_body_success": {"comparator": "eq", "expected": False}, - "resp_headers_contenttype": {"comparator": "eq", "expected": "html/text"} - } + "validators": [ + {"check": "resp_status_code", "comparator": "eq", "expected": 200}, + {"check": "resp_body_success", "comparator": "eq", "expected": False}, + {"check": "resp_headers_contenttype", "comparator": "eq", "expected": "html/text"} + ] } - success, diff_content = self.test_runner.run_test(testcase) + success, diff_content_list = self.test_runner.run_test(testcase) self.assertFalse(success) self.assertEqual( - diff_content['resp_status_code'], - {"comparator": "eq", "expected": 200, 'value': 201} + diff_content_list[0], + {"check": "resp_status_code", "comparator": "eq", "expected": 200, 'value': 201} ) self.assertEqual( - diff_content['resp_body_success'], - {"comparator": "eq", "expected": False, 'value': True} + diff_content_list[1], + {"check": "resp_body_success", "comparator": "eq", "expected": False, 'value': True} ) self.assertEqual( - diff_content['resp_headers_contenttype'], - {"comparator": "eq", "expected": "html/text", 'value': "application/json"} + diff_content_list[2], + {"check": "resp_headers_contenttype", "comparator": "eq", "expected": "html/text", 'value': "application/json"} ) def test_run_testset_json_success(self): @@ -73,25 +73,25 @@ class TestRunner(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testset(testsets[0]) self.assertEqual(len(results), 2) - self.assertEqual(results, [(True, {}), (True, {})]) + self.assertEqual(results, [(True, []), (True, [])]) def test_run_testsets_json_success(self): testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.json') testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) self.assertEqual(len(results), 1) - self.assertEqual(results[0], [(True, {}), (True, {})]) + self.assertEqual(results[0], [(True, []), (True, [])]) def test_run_testset_yaml_success(self): testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml') testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testset(testsets[0]) self.assertEqual(len(results), 2) - self.assertEqual(results, [(True, {}), (True, {})]) + self.assertEqual(results, [(True, []), (True, [])]) def test_run_testsets_yaml_success(self): testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml') testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) self.assertEqual(len(results), 1) - self.assertEqual(results[0], [(True, {}), (True, {})]) + self.assertEqual(results[0], [(True, []), (True, [])]) diff --git a/test/test_runner_v2.py b/test/test_runner_v2.py index c8a764a5..d7ae1b63 100644 --- a/test/test_runner_v2.py +++ b/test/test_runner_v2.py @@ -37,7 +37,7 @@ class TestRunnerV2(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testset(testsets[0]) self.assertEqual(len(results), 2) - self.assertEqual(results, [(True, {}), (True, {})]) + self.assertEqual(results, [(True, []), (True, [])]) def test_run_testsets_auth_yaml(self): testcase_file_path = os.path.join( @@ -45,7 +45,7 @@ class TestRunnerV2(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) self.assertEqual(len(results), 1) - self.assertEqual(results[0], [(True, {}), (True, {})]) + self.assertEqual(results[0], [(True, []), (True, [])]) def test_run_testset_auth_json(self): testcase_file_path = os.path.join( @@ -53,7 +53,7 @@ class TestRunnerV2(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testset(testsets[0]) self.assertEqual(len(results), 2) - self.assertEqual(results, [(True, {}), (True, {})]) + self.assertEqual(results, [(True, []), (True, [])]) def test_run_testsets_auth_json(self): testcase_file_path = os.path.join( @@ -61,7 +61,7 @@ class TestRunnerV2(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) self.assertEqual(len(results), 1) - self.assertEqual(results[0], [(True, {}), (True, {})]) + self.assertEqual(results[0], [(True, []), (True, [])]) def test_run_testcase_template_yaml(self): testcase_file_path = os.path.join( @@ -78,7 +78,7 @@ class TestRunnerV2(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testset(testsets[0]) self.assertEqual(len(results), 2) - self.assertEqual(results, [(True, {}), (True, {})]) + self.assertEqual(results, [(True, []), (True, [])]) def test_run_testsets_template_yaml(self): testcase_file_path = os.path.join( @@ -86,4 +86,4 @@ class TestRunnerV2(ApiServerUnittest): testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) self.assertEqual(len(results), 1) - self.assertEqual(results[0], [(True, {}), (True, {})]) + self.assertEqual(results[0], [(True, []), (True, [])])