diff --git a/httprunner/api.py b/httprunner/api.py index 2d17af46..9fc2e999 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -9,21 +9,27 @@ from httprunner import (exceptions, loader, logger, parser, report, runner, class HttpRunner(object): - def __init__(self, **kwargs): + def __init__(self, failfast=False, save_tests=False, report_template=None, report_dir=None): """ initialize HttpRunner. Args: - kwargs (dict): key-value arguments used to initialize TextTestRunner. - Commonly used arguments: - - failfast (bool): False/True, stop the test run on the first error or failure. + failfast (bool): stop the test run on the first error or failure. + save_tests (bool): save loaded/parsed tests to JSON file. + report_template (str): report template file path, template should be in Jinja2 format. + report_dir (str): html report save directory. """ self.exception_stage = "initialize HttpRunner()" - kwargs["resultclass"] = report.HtmlTestResult + kwargs = { + "failfast": failfast, + "resultclass": report.HtmlTestResult + } self.unittest_runner = unittest.TextTestRunner(**kwargs) self.test_loader = unittest.TestLoader() - self.summary = None + self.save_tests = save_tests + self.report_template = report_template + self.report_dir = report_dir + self._summary = None def _add_tests(self, tests_mapping): """ initialize testcase with Runner() and add to test suite. @@ -108,7 +114,7 @@ class HttpRunner(object): tests_results (list): list of (testcase, result) """ - self.summary = { + summary = { "success": True, "stat": {}, "time": {}, @@ -120,83 +126,98 @@ class HttpRunner(object): testcase, result = tests_result testcase_summary = report.get_summary(result) - self.summary["success"] &= testcase_summary["success"] + summary["success"] &= testcase_summary["success"] testcase_summary["name"] = testcase.config.get("name") in_out = utils.get_testcase_io(testcase) utils.print_io(in_out) testcase_summary["in_out"] = in_out - report.aggregate_stat(self.summary["stat"], testcase_summary["stat"]) - report.aggregate_stat(self.summary["time"], testcase_summary["time"]) + report.aggregate_stat(summary["stat"], testcase_summary["stat"]) + report.aggregate_stat(summary["time"], testcase_summary["time"]) - self.summary["details"].append(testcase_summary) + summary["details"].append(testcase_summary) - def run(self, path_or_testcases, dot_env_path=None, mapping=None, save_tests=False): - """ main interface, run testcases with variables mapping. + return summary + + def run_tests(self, tests_mapping): + """ run testcase/testsuite data + """ + # parse tests + self.exception_stage = "parse tests" + parser.parse_tests(tests_mapping) + + if self.save_tests: + utils.dump_tests(tests_mapping, "parsed") + + # add tests to test suite + self.exception_stage = "add tests to test suite" + test_suite = self._add_tests(tests_mapping) + + # run test suite + self.exception_stage = "run test suite" + results = self._run_suite(test_suite) + + # aggregate results + self.exception_stage = "aggregate results" + self._summary = self._aggregate(results) + + # generate html report + self.exception_stage = "generate html report" + report_path = report.render_html_report( + self._summary, + self.report_template, + self.report_dir + ) + + return report_path + + def run_path(self, path, dot_env_path=None, mapping=None): + """ run testcase/testsuite file or folder. Args: - path_or_testcases (str/list/dict): testcase file/foler path, or valid testcases. + path (str): testcase/testsuite file/foler path. dot_env_path (str): specified .env file path. mapping (dict): if mapping is specified, it will override variables in config block. - save_tests (bool): set if save loaded/parsed tests to JSON file. Returns: instance: HttpRunner() instance """ + # load tests self.exception_stage = "load tests" + tests_mapping = loader.load_tests(path, dot_env_path) + tests_mapping["project_mapping"]["test_path"] = path - if validator.is_testcases(path_or_testcases): - tests_mapping = path_or_testcases - elif validator.is_testcase_path(path_or_testcases): - tests_mapping = loader.load_tests(path_or_testcases, dot_env_path) - tests_mapping["project_mapping"]["test_path"] = path_or_testcases - if "variables" in tests_mapping["project_mapping"]: - tests_mapping["project_mapping"]["variables"] = mapping + if mapping: + tests_mapping["project_mapping"]["variables"] = mapping + + if self.save_tests: + utils.dump_tests(tests_mapping, "loaded") + + return self.run_tests(tests_mapping) + + def run(self, path_or_tests, dot_env_path=None, mapping=None): + """ main interface. + + Args: + path_or_tests: + str: testcase/testsuite file/foler path + dict: valid testcase/testsuite data + + """ + if validator.is_testcases(path_or_tests): + return self.run_tests(path_or_tests) + elif validator.is_testcase_path(path_or_tests): + return self.run_path(path_or_tests, dot_env_path, mapping) else: raise exceptions.ParamsError("invalid testcase path or testcases.") - if save_tests: - utils.dump_tests(tests_mapping, "loaded") - - self.exception_stage = "parse tests" - parser.parse_tests(tests_mapping) - - if save_tests: - utils.dump_tests(tests_mapping, "parsed") - - self.exception_stage = "add tests to test suite" - test_suite = self._add_tests(tests_mapping) - - self.exception_stage = "run test suite" - results = self._run_suite(test_suite) - - self.exception_stage = "aggregate results" - self._aggregate(results) - - return self - - def gen_html_report(self, html_report_name=None, html_report_template=None): - """ generate html report and return report path. - - Args: - html_report_name (str): output html report file name - html_report_template (str): report template file path, template should be in Jinja2 format - - Returns: - str: generated html report path - + @property + def summary(self): + """ get test reuslt summary. """ - if not self.summary: - raise exceptions.MyBaseError("run method should be called before gen_html_report.") - - self.exception_stage = "generate report" - return report.render_html_report( - self.summary, - html_report_name, - html_report_template - ) + return self._summary def prepare_locust_tests(path): diff --git a/httprunner/cli.py b/httprunner/cli.py index 7c928535..bb8ec9d8 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -24,15 +24,6 @@ def main_hrun(): parser.add_argument( 'testcase_paths', nargs='*', help="testcase file path") - parser.add_argument( - '--no-html-report', action='store_true', default=False, - help="do not generate html report.") - parser.add_argument( - '--html-report-name', - help="specify html report name, only effective when generating html report.") - parser.add_argument( - '--html-report-template', - help="specify html report template path.") parser.add_argument( '--log-level', default='INFO', help="Specify logging level, default is INFO.") @@ -42,6 +33,12 @@ def main_hrun(): parser.add_argument( '--dot-env-path', help="Specify .env file path, which is useful for keeping sensitive data.") + parser.add_argument( + '--report-template', + help="specify report template path.") + parser.add_argument( + '--report-dir', + help="specify report save directory.") parser.add_argument( '--failfast', action='store_true', default=False, help="Stop the test run on the first error or failure.") @@ -80,26 +77,20 @@ def main_hrun(): create_scaffold(project_name) exit(0) - for path in args.testcase_paths: - try: - runner = HttpRunner(failfast=args.failfast) - runner.run( - path, - dot_env_path=args.dot_env_path, - save_tests=args.save_tests - ) - except Exception: - logger.log_error("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage)) - raise + runner = HttpRunner( + failfast=args.failfast, + save_tests=args.save_tests, + report_template=args.report_template, + report_dir=args.report_dir + ) + try: + for path in args.testcase_paths: + runner.run(path, dot_env_path=args.dot_env_path) + except Exception: + logger.log_error("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage)) + raise - if not args.no_html_report: - runner.gen_html_report( - html_report_name=args.html_report_name, - html_report_template=args.html_report_template - ) - - summary = runner.summary - return 0 if summary["success"] else 1 + return 0 def main_locust(): """ Performance test with locust: parse command line options and run commands. diff --git a/httprunner/loader.py b/httprunner/loader.py index d03cbde8..6f147a53 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -585,7 +585,7 @@ def load_tests(path, dot_env_path=None): """ load testcases from file path, extend and merge with api/testcase definitions. Args: - path (str/list): testcase file/foler path. + path (str): testcase/testsuite file/foler path. path could be in 2 types: - absolute/relative file path - absolute/relative folder path diff --git a/httprunner/report.py b/httprunner/report.py index af68e429..5450541b 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -78,37 +78,36 @@ def aggregate_stat(origin_stat, new_stat): origin_stat[key] += new_stat[key] -def render_html_report(summary, html_report_name=None, html_report_template=None): +def render_html_report(summary, report_template=None, report_dir=None): """ render html report with specified report name and template - if html_report_name is not specified, use current datetime - if html_report_template is not specified, use default report template + + Args: + report_template (str): specify html report template path + report_dir (str): specify html report save directory + """ - if not html_report_template: - html_report_template = os.path.join( + if not report_template: + report_template = os.path.join( os.path.abspath(os.path.dirname(__file__)), "templates", "report_template.html" ) logger.log_debug("No html report template specified, use default.") else: - logger.log_info("render with html report template: {}".format(html_report_template)) + logger.log_info("render with html report template: {}".format(report_template)) logger.log_info("Start to render Html report ...") logger.log_debug("render data: {}".format(summary)) - report_dir_path = os.path.join(os.getcwd(), "reports") + report_dir = report_dir or os.path.join(os.getcwd(), "reports") + if not os.path.isdir(report_dir): + os.makedirs(report_dir) + start_at_timestamp = int(summary["time"]["start_at"]) summary["time"]["start_datetime"] = datetime.fromtimestamp(start_at_timestamp).strftime('%Y-%m-%d %H:%M:%S') - if html_report_name: - summary["html_report_name"] = html_report_name - report_dir_path = os.path.join(report_dir_path, html_report_name) - html_report_name += "-{}.html".format(start_at_timestamp) - else: - summary["html_report_name"] = "" - html_report_name = "{}.html".format(start_at_timestamp) - - if not os.path.isdir(report_dir_path): - os.makedirs(report_dir_path) + summary["html_report_name"] = "" + html_report_name = "{}.html".format(start_at_timestamp) + report_path = os.path.join(report_dir, html_report_name) for index, suite_summary in enumerate(summary["details"]): if not suite_summary.get("name"): @@ -118,9 +117,8 @@ def render_html_report(summary, html_report_name=None, html_report_template=None stringify_data(meta_data, 'request') stringify_data(meta_data, 'response') - with io.open(html_report_template, "r", encoding='utf-8') as fp_r: + with io.open(report_template, "r", encoding='utf-8') as fp_r: template_content = fp_r.read() - report_path = os.path.join(report_dir_path, html_report_name) with io.open(report_path, 'w', encoding='utf-8') as fp_w: rendered_content = Template( template_content, diff --git a/tests/test_api.py b/tests/test_api.py index a1cb4649..d46ec0a3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -65,6 +65,7 @@ class TestHttpRunner(ApiServerUnittest): self.tests_mapping = { "testcases": testcases } + self.runner = HttpRunner(failfast=True) self.reset_all() def reset_all(self): @@ -73,36 +74,34 @@ class TestHttpRunner(ApiServerUnittest): return self.api_client.get(url, headers=headers) def test_text_run_times(self): - runner = HttpRunner().run(self.testcase_cli_path) - self.assertEqual(runner.summary["stat"]["testsRun"], 10) + self.runner.run(self.testcase_cli_path) + self.assertEqual(self.runner.summary["stat"]["testsRun"], 10) def test_text_skip(self): - runner = HttpRunner().run(self.testcase_cli_path) - self.assertEqual(runner.summary["stat"]["skipped"], 4) + self.runner.run(self.testcase_cli_path) + self.assertEqual(self.runner.summary["stat"]["skipped"], 4) def test_html_report(self): - runner = HttpRunner().run(self.testcase_cli_path) + report_save_dir = os.path.join(os.getcwd(), 'reports', "demo") + runner = HttpRunner(failfast=True, report_dir=report_save_dir) + runner.run(self.testcase_cli_path) summary = runner.summary self.assertEqual(summary["stat"]["testsRun"], 10) self.assertEqual(summary["stat"]["skipped"], 4) - - output_folder_name = "demo" - runner.gen_html_report(html_report_name=output_folder_name) - report_save_dir = os.path.join(os.getcwd(), 'reports', output_folder_name) self.assertGreater(len(os.listdir(report_save_dir)), 0) shutil.rmtree(report_save_dir) def test_run_testcases(self): - runner = HttpRunner().run(self.tests_mapping) - summary = runner.summary + self.runner.run_tests(self.tests_mapping) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 2) self.assertIn("details", summary) self.assertIn("records", summary["details"][0]) def test_run_yaml_upload(self): - runner = HttpRunner().run("tests/httpbin/upload.yml") - summary = runner.summary + self.runner.run("tests/httpbin/upload.yml") + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 1) self.assertIn("details", summary) @@ -140,31 +139,29 @@ class TestHttpRunner(ApiServerUnittest): tests_mapping = { "testcases": testcases } - runner = HttpRunner().run(tests_mapping) - summary = runner.summary + self.runner.run_tests(tests_mapping) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 1) self.assertEqual(summary["details"][0]["records"][0]["meta_data"]["response"]["json"]["data"], "abc") def test_html_report_repsonse_image(self): - runner = HttpRunner().run("tests/httpbin/load_image.yml") - summary = runner.summary - output_folder_name = "demo" - report = runner.gen_html_report(html_report_name=output_folder_name) + report_save_dir = os.path.join(os.getcwd(), 'reports', "demo") + runner = HttpRunner(failfast=True, report_dir=report_save_dir) + report = runner.run("tests/httpbin/load_image.yml") self.assertTrue(os.path.isfile(report)) - report_save_dir = os.path.join(os.getcwd(), 'reports', output_folder_name) shutil.rmtree(report_save_dir) def test_testcase_layer_with_api(self): - runner = HttpRunner(failfast=True).run("tests/testcases/setup.yml") - summary = runner.summary + self.runner.run("tests/testcases/setup.yml") + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["details"][0]["records"][0]["name"], "get token (setup)") self.assertEqual(summary["stat"]["testsRun"], 2) def test_testcase_layer_with_testcase(self): - runner = HttpRunner(failfast=True).run("tests/testsuites/create_users.yml") - summary = runner.summary + self.runner.run("tests/testsuites/create_users.yml") + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 2) @@ -172,9 +169,9 @@ class TestHttpRunner(ApiServerUnittest): testcase_file_path = os.path.join( os.getcwd(), 'tests/httpbin/hooks.yml') start_time = time.time() - runner = HttpRunner().run(testcase_file_path) + self.runner.run(testcase_file_path) end_time = time.time() - summary = runner.summary + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertLess(end_time - start_time, 60) @@ -212,8 +209,8 @@ class TestHttpRunner(ApiServerUnittest): "project_mapping": loader.project_mapping, "testcases": testcases } - runner = HttpRunner().run(tests_mapping) - summary = runner.summary + self.runner.run_tests(tests_mapping) + summary = self.runner.summary self.assertTrue(summary["success"]) def test_run_httprunner_with_teardown_hooks_not_exist_attribute(self): @@ -245,8 +242,8 @@ class TestHttpRunner(ApiServerUnittest): "project_mapping": loader.project_mapping, "testcases": testcases } - runner = HttpRunner().run(tests_mapping) - summary = runner.summary + self.runner.run_tests(tests_mapping) + summary = self.runner.summary self.assertFalse(summary["success"]) self.assertEqual(summary["stat"]["errors"], 1) @@ -276,15 +273,15 @@ class TestHttpRunner(ApiServerUnittest): "project_mapping": loader.project_mapping, "testcases": testcases } - runner = HttpRunner().run(tests_mapping) - summary = runner.summary + self.runner.run_tests(tests_mapping) + summary = self.runner.summary self.assertFalse(summary["success"]) self.assertEqual(summary["stat"]["errors"], 1) def test_run_testcase_hardcode(self): for testcase_file_path in self.testcase_file_path_list: - runner = HttpRunner().run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 3) self.assertEqual(summary["stat"]["successes"], 3) @@ -292,30 +289,30 @@ class TestHttpRunner(ApiServerUnittest): def test_run_testcase_template_variables(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_variables.yml') - runner = HttpRunner().run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary self.assertTrue(summary["success"]) def test_run_testcase_template_import_functions(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_functions.yml') - runner = HttpRunner().run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary self.assertTrue(summary["success"]) def test_run_testcase_layered(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_layer.yml') - runner = HttpRunner().run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(len(summary["details"]), 1) def test_run_testcase_output(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_layer.yml') - runner = HttpRunner(failfast=True).run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertIn("token", summary["details"][0]["in_out"]["out"]) # TODO: add @@ -327,8 +324,8 @@ class TestHttpRunner(ApiServerUnittest): variables_mapping = { "app_version": '2.9.7' } - runner = HttpRunner(failfast=True).run(testcase_file_path, mapping=variables_mapping) - summary = runner.summary + self.runner.run(testcase_file_path, mapping=variables_mapping) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertIn("token", summary["details"][0]["in_out"]["out"]) # TODO: add @@ -337,8 +334,8 @@ class TestHttpRunner(ApiServerUnittest): def test_run_testcase_with_parameters(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_parameters.yml') - runner = HttpRunner().run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary # TODO: add parameterize # self.assertEqual( # summary["details"][0]["in_out"]["in"]["user_agent"], @@ -364,8 +361,7 @@ class TestHttpRunner(ApiServerUnittest): os.getcwd(), 'tests/data/demo_parameters.yml') tests_mapping = loader.load_tests(testcase_file_path) parser.parse_tests(tests_mapping) - runner = HttpRunner() - test_suite = runner._add_tests(tests_mapping) + test_suite = self.runner._add_tests(tests_mapping) self.assertEqual( test_suite._tests[0].tests[0]['name'], @@ -396,8 +392,8 @@ class TestHttpRunner(ApiServerUnittest): def test_validate_response_content(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/httpbin/basic.yml') - runner = HttpRunner().run(testcase_file_path) - self.assertTrue(runner.summary["success"]) + self.runner.run(testcase_file_path) + self.assertTrue(self.runner.summary["success"]) class TestApi(ApiServerUnittest):