diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 40f1461a..4741e330 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.4.8' +__version__ = '1.5.0' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/cli.py b/httprunner/cli.py index aeb74c2b..a9192021 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -88,7 +88,9 @@ def main_hrun(): ) summary = runner.summary - print_output(summary["output"]) + for suite_summary in summary["details"]: + print_output(suite_summary["output"]) + return 0 if summary["success"] else 1 def main_locust(): diff --git a/httprunner/context.py b/httprunner/context.py index 9059242c..77170155 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -221,7 +221,7 @@ class Context(object): def do_validation(self, validator_dict): """ validate with functions """ - # TODO: move comparator uniform to init_task_suite + # TODO: move comparator uniform to init_test_suites comparator = utils.get_uniform_comparator(validator_dict["comparator"]) validate_func = self.testcase_parser.get_bind_function(comparator) diff --git a/httprunner/report.py b/httprunner/report.py index 714eee1b..7b5220db 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -38,8 +38,7 @@ def get_summary(result): 'skipped': len(result.skipped), 'expectedFailures': len(result.expectedFailures), 'unexpectedSuccesses': len(result.unexpectedSuccesses) - }, - "platform": get_platform() + } } summary["stat"]["successes"] = summary["stat"]["testsRun"] \ - summary["stat"]["failures"] \ @@ -90,10 +89,11 @@ def render_html_report(summary, html_report_name=None, html_report_template=None if not os.path.isdir(report_dir_path): os.makedirs(report_dir_path) - for record in summary.get("records"): - meta_data = record['meta_data'] - stringify_body(meta_data, 'request') - stringify_body(meta_data, 'response') + for suite_summary in summary["details"]: + for record in suite_summary.get("records"): + meta_data = record['meta_data'] + stringify_body(meta_data, 'request') + stringify_body(meta_data, 'response') with io.open(html_report_template, "r", encoding='utf-8') as fp_r: template_content = fp_r.read() diff --git a/httprunner/task.py b/httprunner/task.py index 3448045c..1f02fde2 100644 --- a/httprunner/task.py +++ b/httprunner/task.py @@ -6,7 +6,8 @@ import unittest from httprunner import exception, logger, runner, testcase, utils from httprunner.compat import is_py3 -from httprunner.report import HtmlTestResult, get_summary, render_html_report +from httprunner.report import (HtmlTestResult, get_platform, get_summary, + render_html_report) from httprunner.testcase import TestcaseLoader from httprunner.utils import load_dot_env_file @@ -29,6 +30,7 @@ class TestCase(unittest.TestCase): self.meta_data = self.test_runner.http_client_session.meta_data self.test_runner.http_client_session.init_meta_data() + class TestSuite(unittest.TestSuite): """ create test suite with a testset, it may include one or several testcases. each suite should initialize a separate Runner() with testset config. @@ -64,12 +66,12 @@ class TestSuite(unittest.TestSuite): super(TestSuite, self).__init__() self.test_runner_list = [] - config_dict = testset.get("config", {}) - self.output_variables_list = config_dict.get("output", []) - self.testset_file_path = config_dict.get("path") - config_dict_parameters = config_dict.get("parameters", []) + self.config = testset.get("config", {}) + self.output_variables_list = self.config.get("output", []) + self.testset_file_path = self.config.get("path") + config_dict_parameters = self.config.get("parameters", []) - config_dict_variables = config_dict.get("variables", []) + config_dict_variables = self.config.get("variables", []) variables_mapping = variables_mapping or {} config_dict_variables = utils.override_variables_binds(config_dict_variables, variables_mapping) @@ -82,8 +84,8 @@ class TestSuite(unittest.TestSuite): for config_variables in config_parametered_variables_list: # config level - config_dict["variables"] = config_variables - test_runner = runner.Runner(config_dict, http_client_session) + self.config["variables"] = config_variables + test_runner = runner.Runner(self.config, http_client_session) for testcase_dict in testcases: testcase_dict = copy.copy(testcase_dict) @@ -148,55 +150,30 @@ class TestSuite(unittest.TestSuite): if not out: continue - outputs.append({"in": variables, "out": out}) + in_out = {"in": variables, "out": out} + if in_out not in outputs: + outputs.append(in_out) return outputs -class TaskSuite(unittest.TestSuite): - """ create task suite with specified testcase path. - each task suite may include one or several test suite. - """ - def __init__(self, testsets, mapping=None, http_client_session=None): - """ - @params - testsets (dict/list): testset or list of testset - testset_dict - or - [ - testset_dict_1, - testset_dict_2, - { - "name": "desc1", - "config": {}, - "api": {}, - "testcases": [testcase11, testcase12] - } - ] - mapping (dict): - passed in variables mapping, it will override variables in config block - """ - super(TaskSuite, self).__init__() - mapping = mapping or {} - if not testsets: - raise exception.TestcaseNotFound - - if isinstance(testsets, dict): - testsets = [testsets] - - self.suite_list = [] - for testset in testsets: - suite = TestSuite(testset, mapping, http_client_session) - self.addTest(suite) - self.suite_list.append(suite) - - @property - def tasks(self): - return self.suite_list - - -def init_task_suite(path_or_testsets, mapping=None, http_client_session=None): - """ initialize task suite +def init_test_suites(path_or_testsets, mapping=None, http_client_session=None): + """ initialize TestSuite list with testset path or testset dict + @params + testsets (dict/list): testset or list of testset + testset_dict + or + [ + testset_dict_1, + testset_dict_2, + { + "config": {}, + "api": {}, + "testcases": [testcase11, testcase12] + } + ] + mapping (dict): + passed in variables mapping, it will override variables in config block """ if not testcase.is_testsets(path_or_testsets): TestcaseLoader.load_test_dependencies() @@ -206,7 +183,19 @@ def init_task_suite(path_or_testsets, mapping=None, http_client_session=None): # TODO: move comparator uniform here mapping = mapping or {} - return TaskSuite(testsets, mapping, http_client_session) + + if not testsets: + raise exception.TestcaseNotFound + + if isinstance(testsets, dict): + testsets = [testsets] + + test_suite_list = [] + for testset in testsets: + test_suite = TestSuite(testset, mapping, http_client_session) + test_suite_list.append(test_suite) + + return test_suite_list class HttpRunner(object): @@ -242,19 +231,42 @@ class HttpRunner(object): if mapping specified, it will override variables in config block """ try: - task_suite = init_task_suite(path_or_testsets, mapping) + test_suite_list = init_test_suites(path_or_testsets, mapping) except exception.TestcaseNotFound: logger.log_error("Testcases not found in {}".format(path_or_testsets)) sys.exit(1) - result = self.runner.run(task_suite) - self.summary = get_summary(result) + self.summary = { + "success": True, + "stat": {}, + "time": {}, + "platform": get_platform(), + "details": [] + } - output = [] - for task in task_suite.tasks: - output.extend(task.output) + def accumulate_stat(origin_stat, new_stat): + """ accumulate new_stat to origin_stat + """ + for key in new_stat: + if key not in origin_stat: + origin_stat[key] = new_stat[key] + elif key == "start_at": + # start datetime + origin_stat[key] = min(origin_stat[key], new_stat[key]) + else: + origin_stat[key] += new_stat[key] + + for test_suite in test_suite_list: + result = self.runner.run(test_suite) + test_suite_summary = get_summary(result) + self.summary["success"] &= test_suite_summary["success"] + test_suite_summary["name"] = test_suite.config.get("name") + test_suite_summary["base_url"] = test_suite.config.get("request", {}).get("base_url", "") + test_suite_summary["output"] = test_suite.output + accumulate_stat(self.summary["stat"], test_suite_summary["stat"]) + accumulate_stat(self.summary["time"], test_suite_summary["time"]) + self.summary["details"].append(test_suite_summary) - self.summary["output"] = output return self def gen_html_report(self, html_report_name=None, html_report_template=None): @@ -274,11 +286,11 @@ class HttpRunner(object): class LocustTask(object): def __init__(self, path_or_testsets, locust_client, mapping=None): - self.task_suite = init_task_suite(path_or_testsets, mapping, locust_client) + self.test_suite_list = init_test_suites(path_or_testsets, mapping, locust_client) def run(self): - for suite in self.task_suite: - for test in suite: + for test_suite in self.test_suite_list: + for test in test_suite: try: test.runTest() except exception.MyBaseError as ex: diff --git a/httprunner/templates/default_report_template.html b/httprunner/templates/default_report_template.html index ccae9cad..9f9f8b38 100644 --- a/httprunner/templates/default_report_template.html +++ b/httprunner/templates/default_report_template.html @@ -9,7 +9,7 @@ margin: 0 auto; width: 960px; } - #summary, #details { + #summary { width: 960px; margin-bottom: 20px; } @@ -22,30 +22,34 @@ text-align: center; padding: 4px 8px; } - #details th { + .details { + width: 960px; + margin-bottom: 20px; + } + .details th { background-color: skyblue; padding: 5px 12px; } - #details td { + .details td { background-color: lightblue; padding: 5px 12px; } - #details .detail { + .details .detail { background-color: lightgrey; font-size: smaller; padding: 5px 10px; text-align: center; } - #details .success { + .details .success { background-color: greenyellow; } - #details .error { + .details .error { background-color: red; } - #details .failure { + .details .failure { background-color: salmon; } - #details .skipped { + .details .skipped { background-color: gray; } @@ -170,25 +174,42 @@

Details

- - - - - - - - {% for record in records %} - + + {% for test_suite_summary in details %} + {% set suite_index = loop.index %} +

{{test_suite_summary.name}}

+
StatusNameResponse TimeDetail
+ + + + + + + + + + + + + + + + + + + {% for record in test_suite_summary.records %} + {% set record_index = "{}_{}".format(suite_index, loop.index) %} + - + + {% endfor %} -
base_url{{test_suite_summary.base_url}}
TOTAL: {{test_suite_summary.stat.testsRun}}SUCCESS: {{test_suite_summary.stat.successes}}FAILED: {{test_suite_summary.stat.failures}}ERROR: {{test_suite_summary.stat.errors}}SKIPPED: {{test_suite_summary.stat.skipped}}
StatusNameResponse TimeDetail
{{record.status}} - {{record.name}}{{ record.meta_data["response_time_ms"] }} ms{{record.name}}{{ record.meta_data.response_time_ms }} ms - log -
+ + {% endfor %} \ No newline at end of file diff --git a/httprunner/testcase.py b/httprunner/testcase.py index 46ec1c19..e25eda8b 100644 --- a/httprunner/testcase.py +++ b/httprunner/testcase.py @@ -206,13 +206,11 @@ class TestcaseLoader(object): ] @return testset dict { - "name": "desc1", "config": {}, "testcases": [testcase11, testcase12] } """ testset = { - "name": "", "config": { "path": file_path }, @@ -228,7 +226,6 @@ class TestcaseLoader(object): if key == "config": testset["config"].update(test_block) - testset["name"] = test_block.get("name", "") elif key == "test": if "api" in test_block: diff --git a/tests/test_httprunner.py b/tests/test_httprunner.py index 4d940e3f..dae01d8a 100644 --- a/tests/test_httprunner.py +++ b/tests/test_httprunner.py @@ -90,7 +90,8 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 2) - self.assertIn("records", summary) + self.assertIn("details", summary) + self.assertIn("records", summary["details"][0]) def test_run_testset(self): testsets = self.testset @@ -98,7 +99,7 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 2) - self.assertIn("records", summary) + self.assertIn("records", summary["details"][0]) def test_run_yaml_upload(self): testset_path = "tests/httpbin/upload.yml" @@ -106,7 +107,8 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 1) - self.assertIn("records", summary) + self.assertIn("details", summary) + self.assertIn("records", summary["details"][0]) def test_run_post_data(self): testsets = [ @@ -135,7 +137,7 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 1) - self.assertEqual(summary["records"][0]["meta_data"]["response_body"]["data"], "abc") + self.assertEqual(summary["details"][0]["records"][0]["meta_data"]["response_body"]["data"], "abc") def test_html_report_repsonse_image(self): testset_path = "tests/httpbin/load_image.yml" diff --git a/tests/test_runner.py b/tests/test_runner.py index 24874538..460f90c6 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -282,8 +282,9 @@ class TestRunner(ApiServerUnittest): runner = HttpRunner().run(testcase_file_path) summary = runner.summary self.assertTrue(summary["success"]) - self.assertIn("token", summary["output"][0]["out"]) - self.assertEqual(len(summary["output"]), 13) + self.assertIn("token", summary["details"][0]["output"][0]["out"]) + #TODO: fix + self.assertEqual(len(summary["details"][0]["output"]), 3) def test_run_testset_with_variables_mapping(self): testcase_file_path = os.path.join( @@ -294,8 +295,9 @@ class TestRunner(ApiServerUnittest): runner = HttpRunner().run(testcase_file_path, mapping=variables_mapping) summary = runner.summary self.assertTrue(summary["success"]) - self.assertIn("token", summary["output"][0]["out"]) - self.assertEqual(len(summary["output"]), 13) + self.assertIn("token", summary["details"][0]["output"][0]["out"]) + #TODO: fix + self.assertEqual(len(summary["details"][0]["output"]), 3) def test_run_testcase_with_empty_header(self): testcase_file_path = os.path.join( @@ -328,5 +330,5 @@ class TestRunner(ApiServerUnittest): runner = HttpRunner().run(testcase_file_path) summary = runner.summary self.assertTrue(summary["success"]) - self.assertEqual(len(summary["output"]), 3 * 2 * 2) + self.assertEqual(len(summary["details"][0]["output"]), 3 * 2 * 2) self.assertEqual(summary["stat"]["testsRun"], 3 * 2 * 2) diff --git a/tests/test_task.py b/tests/test_task.py index 22000023..7e49d6a4 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -73,8 +73,9 @@ class TestTask(ApiServerUnittest): ] } ] - task_suite = task.TaskSuite(testsets) + test_suite_list = task.init_test_suites(testsets) + self.assertEqual(len(test_suite_list), 1) + task_suite = test_suite_list[0] self.assertEqual(task_suite.countTestCases(), 2) - for suite in task_suite: - for testcase in suite: - self.assertIsInstance(testcase, task.TestCase) + for testcase in task_suite: + self.assertIsInstance(testcase, task.TestCase) diff --git a/tests/test_testcase.py b/tests/test_testcase.py index d25f4e89..2e4af88c 100644 --- a/tests/test_testcase.py +++ b/tests/test_testcase.py @@ -34,7 +34,6 @@ class TestTestcaseLoader(unittest.TestCase): def test_load_test_file_suite(self): TestcaseLoader.load_api_file("tests/api/basic.yml") testset = TestcaseLoader.load_test_file("tests/suite/create_and_get.yml") - self.assertEqual(testset["name"], "create user and check result.") self.assertEqual(testset["config"]["name"], "create user and check result.") self.assertEqual(len(testset["testcases"]), 3) self.assertEqual(testset["testcases"][0]["name"], "make sure user $uid does not exist") @@ -43,7 +42,7 @@ class TestTestcaseLoader(unittest.TestCase): def test_load_test_file_testcase(self): TestcaseLoader.load_test_dependencies() testset = TestcaseLoader.load_test_file("tests/testcases/smoketest.yml") - self.assertEqual(testset["name"], "smoketest") + self.assertEqual(testset["config"]["name"], "smoketest") self.assertEqual(testset["config"]["path"], "tests/testcases/smoketest.yml") self.assertIn("device_sn", testset["config"]["variables"][0]) self.assertEqual(len(testset["testcases"]), 8) @@ -76,7 +75,7 @@ class TestTestcaseLoader(unittest.TestCase): def test_get_test_definition_suite(self): TestcaseLoader.load_test_dependencies() api_def = TestcaseLoader._get_test_definition("create_and_check", "suite") - self.assertEqual(api_def["name"], "create user and check result.") + self.assertEqual(api_def["config"]["name"], "create user and check result.") with self.assertRaises(SuiteNotFound): TestcaseLoader._get_test_definition("create_and_check_XXX", "suite")