diff --git a/ate/runner.py b/ate/runner.py index 43ba8892..25c1330e 100644 --- a/ate/runner.py +++ b/ate/runner.py @@ -144,22 +144,30 @@ class Runner(object): testcase12 ] } - @return (list) test results of testcases - [ - True, # testcase11 - True # testcase12 - ] + @return (dict) test result of testcases + { + "success": True, + "output": {} # variables mapping + } """ - results = [] - + success = True config_dict = testset.get("config", {}) self.init_config(config_dict, level="testset") testcases = testset.get("testcases", []) for testcase in testcases: - result = self.run_test(testcase) - results.append(result) + try: + assert self.run_test(testcase) + except AssertionError: + success = False - return results + output_variables_list = config_dict.get("output", []) + output = self.generate_output(output_variables_list) + self.print_output(output) + + return { + "success": success, + "output": output + } def run_testsets(self, testsets): """ run testsets, including one or several testsets. @@ -168,16 +176,34 @@ class Runner(object): testset1, testset2, ] - @return (list) test results of testsets - [ - [ # testset1 - True, # testcase11 - True # testcase12 - ], - [ # testset2 - True, # testcase21 - True # testcase22 - ] - ] + @return (bool) test result of testsets """ - return [self.run_testset(testset) for testset in testsets] + success = True + for testset in testsets: + try: + result = self.run_testset(testset) + assert result["success"] + except AssertionError: + success = False + + return success + + def generate_output(self, output_variables_list): + variables_mapping = self.context.get_testcase_variables_mapping() + return { + variable: variables_mapping[variable] + for variable in output_variables_list + } + + def print_output(self, output): + if not output: + return + + print("\n================== Output ==================") + print('{:<10}: {:<}'.format("Variable", "Value")) + print('{:<10}: {:<}'.format("--------", "-----")) + + for variable, value in output.items(): + print('{:<10}: {:<}'.format(variable, value)) + + print("============================================\n") diff --git a/tests/data/demo_testset_layer.yml b/tests/data/demo_testset_layer.yml index ed107f93..87758abf 100644 --- a/tests/data/demo_testset_layer.yml +++ b/tests/data/demo_testset_layer.yml @@ -10,6 +10,8 @@ headers: Content-Type: application/json device_sn: $device_sn + output: + - token - test: name: get token diff --git a/tests/test_runner.py b/tests/test_runner.py index 0f30d28e..be8ceea2 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -68,53 +68,54 @@ class TestRunner(ApiServerUnittest): def test_run_testset_hardcode(self): for testcase_file_path in self.testcase_file_path_list: testsets = load_testcases_by_path(testcase_file_path) - results = self.test_runner.run_testset(testsets[0]) - self.assertEqual(len(results), 3) - self.assertEqual(results, [True] * 3) + result = self.test_runner.run_testset(testsets[0]) + self.assertTrue(result["success"]) def test_run_testsets_hardcode(self): for testcase_file_path in self.testcase_file_path_list: testsets = load_testcases_by_path(testcase_file_path) - results = self.test_runner.run_testsets(testsets) - self.assertEqual(len(results), 1) - self.assertEqual(results, [[True] * 3]) + result = self.test_runner.run_testsets(testsets) + self.assertTrue(result) def test_run_testset_template_variables(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testset_variables.yml') testsets = load_testcases_by_path(testcase_file_path) - results = self.test_runner.run_testset(testsets[0]) - self.assertEqual(len(results), 3) - self.assertEqual(results, [True] * 3) + result = self.test_runner.run_testset(testsets[0]) + self.assertTrue(result["success"]) def test_run_testset_template_import_functions(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testset_template_import_functions.yml') testsets = load_testcases_by_path(testcase_file_path) - results = self.test_runner.run_testset(testsets[0]) - self.assertEqual(len(results), 3) - self.assertEqual(results, [True] * 3) + result = self.test_runner.run_testset(testsets[0]) + self.assertTrue(result["success"]) def test_run_testsets_template_import_functions(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testset_template_import_functions.yml') testsets = load_testcases_by_path(testcase_file_path) - results = self.test_runner.run_testsets(testsets) - self.assertEqual(len(results), 1) - self.assertEqual(results, [[True] * 3]) + result = self.test_runner.run_testsets(testsets) + self.assertTrue(result) def test_run_testsets_template_lambda_functions(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testset_template_lambda_functions.yml') testsets = load_testcases_by_path(testcase_file_path) - results = self.test_runner.run_testsets(testsets) - self.assertEqual(len(results), 1) - self.assertEqual(results, [[True] * 3]) + result = self.test_runner.run_testsets(testsets) + self.assertTrue(result) def test_run_testset_layered(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testset_layer.yml') testsets = load_testcases_by_path(testcase_file_path) - results = self.test_runner.run_testsets(testsets) - self.assertEqual(len(results), 1) - self.assertEqual(results, [[True] * 3]) + result = self.test_runner.run_testsets(testsets) + self.assertTrue(result) + + def test_run_testset_output(self): + testcase_file_path = os.path.join( + os.getcwd(), 'tests/data/demo_testset_layer.yml') + testsets = load_testcases_by_path(testcase_file_path) + result = self.test_runner.run_testset(testsets[0]) + self.assertTrue(result["success"]) + self.assertIn("token", result["output"])