diff --git a/ate/runner.py b/ate/runner.py index 0ed65764..ea8650a1 100644 --- a/ate/runner.py +++ b/ate/runner.py @@ -16,6 +16,7 @@ class TestRunner(object): """ create/update variables binds @param config_dict { + "name": "description content", "requires": ["random", "hashlib"], "function_binds": { "gen_random_string": \ @@ -30,11 +31,6 @@ class TestRunner(object): {"random": {"func": "gen_random_string", "args": [5]}}, ] } - @return variables binds mapping - { - "TOKEN": "debugtalk", - "random": "A2dEx" - } """ requires = config_dict.get('requires', []) self.context.import_requires(requires) @@ -49,15 +45,7 @@ class TestRunner(object): def parse_testcase(self, testcase): """ parse testcase with variables binds if it is a template. - """ - self.pre_config(testcase) - - parsed_testcase = self.testcase_parser.parse(testcase) - return parsed_testcase - - def run_test(self, testcase): - """ run single testcase. - @testcase + @param (dict) testcase { "name": "testcase description", "requires": [], # optional, override @@ -66,6 +54,30 @@ class TestRunner(object): "request": {}, "response": {} } + @return (dict) variables binds mapping + { + "TOKEN": "debugtalk", + "random": "A2dEx" + } + """ + self.pre_config(testcase) + + parsed_testcase = self.testcase_parser.parse(testcase) + return parsed_testcase + + def run_test(self, testcase): + """ run single testcase. + @param (dict) testcase + { + "name": "testcase description", + "requires": [], # optional, override + "function_binds": {}, # optional, override + "variable_binds": {}, # optional, override + "request": {}, + "response": {} + } + @return (tuple) test result of single testcase + (success, diff_content) """ testcase = self.parse_testcase(testcase) @@ -82,42 +94,61 @@ class TestRunner(object): success = False if diff_content else True return success, diff_content - def run_testsets(self, testsets): - """ run testcase suite. - @testsets - [ - { + def run_testset(self, testset): + """ run single testset, including one or several testcases. + @param (dict) testset + { + "name": "testset description", + "config": { "name": "testset description", - "config": { - "requires": [], - "function_binds": {}, - "variable_binds": [] + "requires": [], + "function_binds": {}, + "variable_binds": [] + }, + "testcases": [ + { + "name": "testcase description", + "variable_binds": {}, # override + "request": {}, + "response": {} }, - "testcases": [ - { - "name": "testcase description", - "variable_binds": {}, # override - "request": {}, - "response": {} - }, - testcase12 - ] - }, - { - "name": "XXX", - "config": {}, - "testcases": [testcase21, testcase22, testcase23] - }, + testcase12 + ] + } + @return (list) test results of testcases + [ + (success, diff_content), # testcase1 + (success, diff_content) # testcase2 ] """ results = [] - for testset in testsets: - config_dict = testset.get("config", {}) - self.pre_config(config_dict) - testcases = testset.get("testcases", []) - for testcase in testcases: - result = self.run_test(testcase) - results.append(result) + config_dict = testset.get("config", {}) + self.pre_config(config_dict) + testcases = testset.get("testcases", []) + for testcase in testcases: + result = self.run_test(testcase) + results.append(result) return results + + def run_testsets(self, testsets): + """ run testsets, including one or several testsets. + @param testsets + [ + testset1, + testset2, + ] + @return (list) test results of testsets + [ + [ # testset1 + (success, diff_content), # testcase11 + (success, diff_content) # testcase12 + ], + [ # testset2 + (success, diff_content), # testcase21 + (success, diff_content) # testcase22 + ] + ] + """ + return [self.run_testset(testset) for testset in testsets] diff --git a/ate/utils.py b/ate/utils.py index 59eddd65..a42e313d 100644 --- a/ate/utils.py +++ b/ate/utils.py @@ -147,8 +147,8 @@ def load_testcases_by_path(path): - list/set container with file(s) and/or folder(s) @return testcase sets list, each testset is corresponding to a file [ - {"config": {}, "testcases": [testcase11, testcase12]}, - {"config": {}, "testcases": [testcase21, testcase22, testcase23]}, + {"name": "desc1", "config": {}, "testcases": [testcase11, testcase12]}, + {"name": "desc2", "config": {}, "testcases": [testcase21, testcase22, testcase23]}, ] """ if isinstance(path, (list, set)): @@ -169,6 +169,7 @@ def load_testcases_by_path(path): if os.path.isfile(path): testset = { + "name": "", "config": {}, "testcases": [] } @@ -178,6 +179,7 @@ def load_testcases_by_path(path): for key in item: if key == "config": testset["config"] = item["config"] + testset["name"] = item["config"].get("name", "") elif key == "test": testset["testcases"].append(item["test"]) diff --git a/test/test_runner.py b/test/test_runner.py index a29c58d3..595d9a46 100644 --- a/test/test_runner.py +++ b/test/test_runner.py @@ -76,16 +76,30 @@ class TestRunner(ApiServerUnittest): } ) - def test_run_testcase_suite_json_success(self): + def test_run_testset_json_success(self): testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.json') testsets = utils.load_testcases_by_path(testcase_file_path) - results = self.test_runner.run_testsets(testsets) + results = self.test_runner.run_testset(testsets[0]) self.assertEqual(len(results), 2) self.assertEqual(results, [(True, {}), (True, {})]) - def test_run_testcase_suite_yaml_success(self): + def test_run_testsets_json_success(self): + testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.json') + testsets = utils.load_testcases_by_path(testcase_file_path) + results = self.test_runner.run_testsets(testsets) + self.assertEqual(len(results), 1) + self.assertEqual(results[0], [(True, {}), (True, {})]) + + def test_run_testset_yaml_success(self): + testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml') + testsets = utils.load_testcases_by_path(testcase_file_path) + results = self.test_runner.run_testset(testsets[0]) + self.assertEqual(len(results), 2) + self.assertEqual(results, [(True, {}), (True, {})]) + + def test_run_testsets_yaml_success(self): testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml') testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) - self.assertEqual(len(results), 2) - self.assertEqual(results, [(True, {}), (True, {})]) + self.assertEqual(len(results), 1) + self.assertEqual(results[0], [(True, {}), (True, {})]) diff --git a/test/test_runner_v2.py b/test/test_runner_v2.py index f2e3a473..b80d951d 100644 --- a/test/test_runner_v2.py +++ b/test/test_runner_v2.py @@ -31,22 +31,37 @@ class TestRunnerV2(ApiServerUnittest): success, _ = self.test_runner.run_test(testcase) self.assertTrue(success) - def test_run_testcase_auth_suite_yaml(self): + def test_run_testset_auth_yaml(self): + testcase_file_path = os.path.join( + os.getcwd(), 'test/data/simple_demo_auth_hardcode.yml') + testsets = utils.load_testcases_by_path(testcase_file_path) + results = self.test_runner.run_testset(testsets[0]) + self.assertEqual(len(results), 2) + self.assertEqual(results, [(True, {}), (True, {})]) + + def test_run_testsets_auth_yaml(self): testcase_file_path = os.path.join( os.getcwd(), 'test/data/simple_demo_auth_hardcode.yml') testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) + self.assertEqual(len(results), 1) + self.assertEqual(results[0], [(True, {}), (True, {})]) + + def test_run_testset_auth_json(self): + testcase_file_path = os.path.join( + os.getcwd(), 'test/data/simple_demo_auth_hardcode.json') + testsets = utils.load_testcases_by_path(testcase_file_path) + results = self.test_runner.run_testset(testsets[0]) self.assertEqual(len(results), 2) self.assertEqual(results, [(True, {}), (True, {})]) - - def test_run_testcase_auth_suite_json(self): + def test_run_testsets_auth_json(self): testcase_file_path = os.path.join( os.getcwd(), 'test/data/simple_demo_auth_hardcode.json') testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) - self.assertEqual(len(results), 2) - self.assertEqual(results, [(True, {}), (True, {})]) + self.assertEqual(len(results), 1) + self.assertEqual(results[0], [(True, {}), (True, {})]) def test_run_testcase_template_yaml(self): testcase_file_path = os.path.join( @@ -57,10 +72,18 @@ class TestRunnerV2(ApiServerUnittest): success, _ = self.test_runner.run_test(testcases[1]["test"]) self.assertTrue(success) - def test_run_testcase_template_sets_yaml(self): + def test_run_testset_template_yaml(self): + testcase_file_path = os.path.join( + os.getcwd(), 'test/data/demo_template_sets.yml') + testsets = utils.load_testcases_by_path(testcase_file_path) + results = self.test_runner.run_testset(testsets[0]) + self.assertEqual(len(results), 2) + self.assertEqual(results, [(True, {}), (True, {})]) + + def test_run_testsets_template_yaml(self): testcase_file_path = os.path.join( os.getcwd(), 'test/data/demo_template_sets.yml') testsets = utils.load_testcases_by_path(testcase_file_path) results = self.test_runner.run_testsets(testsets) - self.assertEqual(len(results), 2) - self.assertEqual(results, [(True, {}), (True, {})]) + self.assertEqual(len(results), 1) + self.assertEqual(results[0], [(True, {}), (True, {})])