adjust arguments for html report

httprunner/api.py

@@ -9,21 +9,27 @@ from httprunner import (exceptions, loader, logger, parser, report, runner,
 
 class HttpRunner(object):
 
-    def __init__(self, **kwargs):
+    def __init__(self, failfast=False, save_tests=False, report_template=None, report_dir=None):
         """ initialize HttpRunner.
 
         Args:
-            kwargs (dict): key-value arguments used to initialize TextTestRunner.
+            failfast (bool): stop the test run on the first error or failure.
-            Commonly used arguments:
+            save_tests (bool): save loaded/parsed tests to JSON file.
-
+            report_template (str): report template file path, template should be in Jinja2 format.
-            failfast (bool): False/True, stop the test run on the first error or failure.
+            report_dir (str): html report save directory.
 
         """
         self.exception_stage = "initialize HttpRunner()"
-        kwargs["resultclass"] = report.HtmlTestResult
+        kwargs = {
+            "failfast": failfast,
+            "resultclass": report.HtmlTestResult
+        }
         self.unittest_runner = unittest.TextTestRunner(**kwargs)
         self.test_loader = unittest.TestLoader()
-        self.summary = None
+        self.save_tests = save_tests
+        self.report_template = report_template
+        self.report_dir = report_dir
+        self._summary = None
 
     def _add_tests(self, tests_mapping):
         """ initialize testcase with Runner() and add to test suite.
@@ -108,7 +114,7 @@ class HttpRunner(object):
             tests_results (list): list of (testcase, result)
 
         """
-        self.summary = {
+        summary = {
             "success": True,
             "stat": {},
             "time": {},
@@ -120,83 +126,98 @@ class HttpRunner(object):
             testcase, result = tests_result
             testcase_summary = report.get_summary(result)
 
-            self.summary["success"] &= testcase_summary["success"]
+            summary["success"] &= testcase_summary["success"]
             testcase_summary["name"] = testcase.config.get("name")
 
             in_out = utils.get_testcase_io(testcase)
             utils.print_io(in_out)
             testcase_summary["in_out"] = in_out
 
-            report.aggregate_stat(self.summary["stat"], testcase_summary["stat"])
+            report.aggregate_stat(summary["stat"], testcase_summary["stat"])
-            report.aggregate_stat(self.summary["time"], testcase_summary["time"])
+            report.aggregate_stat(summary["time"], testcase_summary["time"])
 
-            self.summary["details"].append(testcase_summary)
+            summary["details"].append(testcase_summary)
 
-    def run(self, path_or_testcases, dot_env_path=None, mapping=None, save_tests=False):
+        return summary
-        """ main interface, run testcases with variables mapping.
+
+    def run_tests(self, tests_mapping):
+        """ run testcase/testsuite data
+        """
+        # parse tests
+        self.exception_stage = "parse tests"
+        parser.parse_tests(tests_mapping)
+
+        if self.save_tests:
+            utils.dump_tests(tests_mapping, "parsed")
+
+        # add tests to test suite
+        self.exception_stage = "add tests to test suite"
+        test_suite = self._add_tests(tests_mapping)
+
+        # run test suite
+        self.exception_stage = "run test suite"
+        results = self._run_suite(test_suite)
+
+        # aggregate results
+        self.exception_stage = "aggregate results"
+        self._summary = self._aggregate(results)
+
+        # generate html report
+        self.exception_stage = "generate html report"
+        report_path = report.render_html_report(
+            self._summary,
+            self.report_template,
+            self.report_dir
+        )
+
+        return report_path
+
+    def run_path(self, path, dot_env_path=None, mapping=None):
+        """ run testcase/testsuite file or folder.
 
         Args:
-            path_or_testcases (str/list/dict): testcase file/foler path, or valid testcases.
+            path (str): testcase/testsuite file/foler path.
             dot_env_path (str): specified .env file path.
             mapping (dict): if mapping is specified, it will override variables in config block.
-            save_tests (bool): set if save loaded/parsed tests to JSON file.
 
         Returns:
             instance: HttpRunner() instance
 
         """
+        # load tests
         self.exception_stage = "load tests"
+        tests_mapping = loader.load_tests(path, dot_env_path)
+        tests_mapping["project_mapping"]["test_path"] = path
 
-        if validator.is_testcases(path_or_testcases):
+        if mapping:
-            tests_mapping = path_or_testcases
+            tests_mapping["project_mapping"]["variables"] = mapping
-        elif validator.is_testcase_path(path_or_testcases):
+
-            tests_mapping = loader.load_tests(path_or_testcases, dot_env_path)
+        if self.save_tests:
-            tests_mapping["project_mapping"]["test_path"] = path_or_testcases
+            utils.dump_tests(tests_mapping, "loaded")
-            if "variables" in tests_mapping["project_mapping"]:
+
-                tests_mapping["project_mapping"]["variables"] = mapping
+        return self.run_tests(tests_mapping)
+
+    def run(self, path_or_tests, dot_env_path=None, mapping=None):
+        """ main interface.
+
+        Args:
+            path_or_tests:
+                str: testcase/testsuite file/foler path
+                dict: valid testcase/testsuite data
+
+        """
+        if validator.is_testcases(path_or_tests):
+            return self.run_tests(path_or_tests)
+        elif validator.is_testcase_path(path_or_tests):
+            return self.run_path(path_or_tests, dot_env_path, mapping)
         else:
             raise exceptions.ParamsError("invalid testcase path or testcases.")
 
-        if save_tests:
+    @property
-            utils.dump_tests(tests_mapping, "loaded")
+    def summary(self):
-
+        """ get test reuslt summary.
-        self.exception_stage = "parse tests"
-        parser.parse_tests(tests_mapping)
-
-        if save_tests:
-            utils.dump_tests(tests_mapping, "parsed")
-
-        self.exception_stage = "add tests to test suite"
-        test_suite = self._add_tests(tests_mapping)
-
-        self.exception_stage = "run test suite"
-        results = self._run_suite(test_suite)
-
-        self.exception_stage = "aggregate results"
-        self._aggregate(results)
-
-        return self
-
-    def gen_html_report(self, html_report_name=None, html_report_template=None):
-        """ generate html report and return report path.
-
-        Args:
-            html_report_name (str): output html report file name
-            html_report_template (str): report template file path, template should be in Jinja2 format
-
-        Returns:
-            str: generated html report path
-
         """
-        if not self.summary:
+        return self._summary
-            raise exceptions.MyBaseError("run method should be called before gen_html_report.")
-
-        self.exception_stage = "generate report"
-        return report.render_html_report(
-            self.summary,
-            html_report_name,
-            html_report_template
-        )
 
 
 def prepare_locust_tests(path):

httprunner/cli.py

@@ -24,15 +24,6 @@ def main_hrun():
     parser.add_argument(
         'testcase_paths', nargs='*',
         help="testcase file path")
-    parser.add_argument(
-        '--no-html-report', action='store_true', default=False,
-        help="do not generate html report.")
-    parser.add_argument(
-        '--html-report-name',
-        help="specify html report name, only effective when generating html report.")
-    parser.add_argument(
-        '--html-report-template',
-        help="specify html report template path.")
     parser.add_argument(
         '--log-level', default='INFO',
         help="Specify logging level, default is INFO.")
@@ -42,6 +33,12 @@ def main_hrun():
     parser.add_argument(
         '--dot-env-path',
         help="Specify .env file path, which is useful for keeping sensitive data.")
+    parser.add_argument(
+        '--report-template',
+        help="specify report template path.")
+    parser.add_argument(
+        '--report-dir',
+        help="specify report save directory.")
     parser.add_argument(
         '--failfast', action='store_true', default=False,
         help="Stop the test run on the first error or failure.")
@@ -80,26 +77,20 @@ def main_hrun():
         create_scaffold(project_name)
         exit(0)
 
-    for path in args.testcase_paths:
+    runner = HttpRunner(
-        try:
+        failfast=args.failfast,
-            runner = HttpRunner(failfast=args.failfast)
+        save_tests=args.save_tests,
-            runner.run(
+        report_template=args.report_template,
-                path,
+        report_dir=args.report_dir
-                dot_env_path=args.dot_env_path,
+    )
-                save_tests=args.save_tests
+    try:
-            )
+        for path in args.testcase_paths:
-        except Exception:
+            runner.run(path, dot_env_path=args.dot_env_path)
-            logger.log_error("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage))
+    except Exception:
-            raise
+        logger.log_error("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage))
+        raise
 
-    if not args.no_html_report:
+    return 0
-        runner.gen_html_report(
-            html_report_name=args.html_report_name,
-            html_report_template=args.html_report_template
-        )
-
-    summary = runner.summary
-    return 0 if summary["success"] else 1
 
 def main_locust():
     """ Performance test with locust: parse command line options and run commands.

httprunner/loader.py

@@ -585,7 +585,7 @@ def load_tests(path, dot_env_path=None):
     """ load testcases from file path, extend and merge with api/testcase definitions.
 
     Args:
-        path (str/list): testcase file/foler path.
+        path (str): testcase/testsuite file/foler path.
             path could be in 2 types:
                 - absolute/relative file path
                 - absolute/relative folder path

httprunner/report.py

@@ -78,37 +78,36 @@ def aggregate_stat(origin_stat, new_stat):
             origin_stat[key] += new_stat[key]
 
 
-def render_html_report(summary, html_report_name=None, html_report_template=None):
+def render_html_report(summary, report_template=None, report_dir=None):
     """ render html report with specified report name and template
-        if html_report_name is not specified, use current datetime
+
-        if html_report_template is not specified, use default report template
+    Args:
+        report_template (str): specify html report template path
+        report_dir (str): specify html report save directory
+
     """
-    if not html_report_template:
+    if not report_template:
-        html_report_template = os.path.join(
+        report_template = os.path.join(
             os.path.abspath(os.path.dirname(__file__)),
             "templates",
             "report_template.html"
         )
         logger.log_debug("No html report template specified, use default.")
     else:
-        logger.log_info("render with html report template: {}".format(html_report_template))
+        logger.log_info("render with html report template: {}".format(report_template))
 
     logger.log_info("Start to render Html report ...")
     logger.log_debug("render data: {}".format(summary))
 
-    report_dir_path = os.path.join(os.getcwd(), "reports")
+    report_dir = report_dir or os.path.join(os.getcwd(), "reports")
+    if not os.path.isdir(report_dir):
+        os.makedirs(report_dir)
+
     start_at_timestamp = int(summary["time"]["start_at"])
     summary["time"]["start_datetime"] = datetime.fromtimestamp(start_at_timestamp).strftime('%Y-%m-%d %H:%M:%S')
-    if html_report_name:
+    summary["html_report_name"] = ""
-        summary["html_report_name"] = html_report_name
+    html_report_name = "{}.html".format(start_at_timestamp)
-        report_dir_path = os.path.join(report_dir_path, html_report_name)
+    report_path = os.path.join(report_dir, html_report_name)
-        html_report_name += "-{}.html".format(start_at_timestamp)
-    else:
-        summary["html_report_name"] = ""
-        html_report_name = "{}.html".format(start_at_timestamp)
-
-    if not os.path.isdir(report_dir_path):
-        os.makedirs(report_dir_path)
 
     for index, suite_summary in enumerate(summary["details"]):
         if not suite_summary.get("name"):
@@ -118,9 +117,8 @@ def render_html_report(summary, html_report_name=None, html_report_template=None
             stringify_data(meta_data, 'request')
             stringify_data(meta_data, 'response')
 
-    with io.open(html_report_template, "r", encoding='utf-8') as fp_r:
+    with io.open(report_template, "r", encoding='utf-8') as fp_r:
         template_content = fp_r.read()
-        report_path = os.path.join(report_dir_path, html_report_name)
         with io.open(report_path, 'w', encoding='utf-8') as fp_w:
             rendered_content = Template(
                 template_content,

tests/test_api.py

@@ -65,6 +65,7 @@ class TestHttpRunner(ApiServerUnittest):
         self.tests_mapping = {
             "testcases": testcases
         }
+        self.runner = HttpRunner(failfast=True)
         self.reset_all()
 
     def reset_all(self):
@@ -73,36 +74,34 @@ class TestHttpRunner(ApiServerUnittest):
         return self.api_client.get(url, headers=headers)
 
     def test_text_run_times(self):
-        runner = HttpRunner().run(self.testcase_cli_path)
+        self.runner.run(self.testcase_cli_path)
-        self.assertEqual(runner.summary["stat"]["testsRun"], 10)
+        self.assertEqual(self.runner.summary["stat"]["testsRun"], 10)
 
     def test_text_skip(self):
-        runner = HttpRunner().run(self.testcase_cli_path)
+        self.runner.run(self.testcase_cli_path)
-        self.assertEqual(runner.summary["stat"]["skipped"], 4)
+        self.assertEqual(self.runner.summary["stat"]["skipped"], 4)
 
     def test_html_report(self):
-        runner = HttpRunner().run(self.testcase_cli_path)
+        report_save_dir = os.path.join(os.getcwd(), 'reports', "demo")
+        runner = HttpRunner(failfast=True, report_dir=report_save_dir)
+        runner.run(self.testcase_cli_path)
         summary = runner.summary
         self.assertEqual(summary["stat"]["testsRun"], 10)
         self.assertEqual(summary["stat"]["skipped"], 4)
-
-        output_folder_name = "demo"
-        runner.gen_html_report(html_report_name=output_folder_name)
-        report_save_dir = os.path.join(os.getcwd(), 'reports', output_folder_name)
         self.assertGreater(len(os.listdir(report_save_dir)), 0)
         shutil.rmtree(report_save_dir)
 
     def test_run_testcases(self):
-        runner = HttpRunner().run(self.tests_mapping)
+        self.runner.run_tests(self.tests_mapping)
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
         self.assertEqual(summary["stat"]["testsRun"], 2)
         self.assertIn("details", summary)
         self.assertIn("records", summary["details"][0])
 
     def test_run_yaml_upload(self):
-        runner = HttpRunner().run("tests/httpbin/upload.yml")
+        self.runner.run("tests/httpbin/upload.yml")
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
         self.assertEqual(summary["stat"]["testsRun"], 1)
         self.assertIn("details", summary)
@@ -140,31 +139,29 @@ class TestHttpRunner(ApiServerUnittest):
         tests_mapping = {
             "testcases": testcases
         }
-        runner = HttpRunner().run(tests_mapping)
+        self.runner.run_tests(tests_mapping)
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
         self.assertEqual(summary["stat"]["testsRun"], 1)
         self.assertEqual(summary["details"][0]["records"][0]["meta_data"]["response"]["json"]["data"], "abc")
 
     def test_html_report_repsonse_image(self):
-        runner = HttpRunner().run("tests/httpbin/load_image.yml")
+        report_save_dir = os.path.join(os.getcwd(), 'reports', "demo")
-        summary = runner.summary
+        runner = HttpRunner(failfast=True, report_dir=report_save_dir)
-        output_folder_name = "demo"
+        report = runner.run("tests/httpbin/load_image.yml")
-        report = runner.gen_html_report(html_report_name=output_folder_name)
         self.assertTrue(os.path.isfile(report))
-        report_save_dir = os.path.join(os.getcwd(), 'reports', output_folder_name)
         shutil.rmtree(report_save_dir)
 
     def test_testcase_layer_with_api(self):
-        runner = HttpRunner(failfast=True).run("tests/testcases/setup.yml")
+        self.runner.run("tests/testcases/setup.yml")
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
         self.assertEqual(summary["details"][0]["records"][0]["name"], "get token (setup)")
         self.assertEqual(summary["stat"]["testsRun"], 2)
 
     def test_testcase_layer_with_testcase(self):
-        runner = HttpRunner(failfast=True).run("tests/testsuites/create_users.yml")
+        self.runner.run("tests/testsuites/create_users.yml")
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
         self.assertEqual(summary["stat"]["testsRun"], 2)
 
@@ -172,9 +169,9 @@ class TestHttpRunner(ApiServerUnittest):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/httpbin/hooks.yml')
         start_time = time.time()
-        runner = HttpRunner().run(testcase_file_path)
+        self.runner.run(testcase_file_path)
         end_time = time.time()
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
         self.assertLess(end_time - start_time, 60)
 
@@ -212,8 +209,8 @@ class TestHttpRunner(ApiServerUnittest):
             "project_mapping": loader.project_mapping,
             "testcases": testcases
         }
-        runner = HttpRunner().run(tests_mapping)
+        self.runner.run_tests(tests_mapping)
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
 
     def test_run_httprunner_with_teardown_hooks_not_exist_attribute(self):
@@ -245,8 +242,8 @@ class TestHttpRunner(ApiServerUnittest):
             "project_mapping": loader.project_mapping,
             "testcases": testcases
         }
-        runner = HttpRunner().run(tests_mapping)
+        self.runner.run_tests(tests_mapping)
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertFalse(summary["success"])
         self.assertEqual(summary["stat"]["errors"], 1)
 
@@ -276,15 +273,15 @@ class TestHttpRunner(ApiServerUnittest):
             "project_mapping": loader.project_mapping,
             "testcases": testcases
         }
-        runner = HttpRunner().run(tests_mapping)
+        self.runner.run_tests(tests_mapping)
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertFalse(summary["success"])
         self.assertEqual(summary["stat"]["errors"], 1)
 
     def test_run_testcase_hardcode(self):
         for testcase_file_path in self.testcase_file_path_list:
-            runner = HttpRunner().run(testcase_file_path)
+            self.runner.run(testcase_file_path)
-            summary = runner.summary
+            summary = self.runner.summary
             self.assertTrue(summary["success"])
             self.assertEqual(summary["stat"]["testsRun"], 3)
             self.assertEqual(summary["stat"]["successes"], 3)
@@ -292,30 +289,30 @@ class TestHttpRunner(ApiServerUnittest):
     def test_run_testcase_template_variables(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/data/demo_testcase_variables.yml')
-        runner = HttpRunner().run(testcase_file_path)
+        self.runner.run(testcase_file_path)
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
 
     def test_run_testcase_template_import_functions(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/data/demo_testcase_functions.yml')
-        runner = HttpRunner().run(testcase_file_path)
+        self.runner.run(testcase_file_path)
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
 
     def test_run_testcase_layered(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/data/demo_testcase_layer.yml')
-        runner = HttpRunner().run(testcase_file_path)
+        self.runner.run(testcase_file_path)
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
         self.assertEqual(len(summary["details"]), 1)
 
     def test_run_testcase_output(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/data/demo_testcase_layer.yml')
-        runner = HttpRunner(failfast=True).run(testcase_file_path)
+        self.runner.run(testcase_file_path)
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
         self.assertIn("token", summary["details"][0]["in_out"]["out"])
         # TODO: add
@@ -327,8 +324,8 @@ class TestHttpRunner(ApiServerUnittest):
         variables_mapping = {
             "app_version": '2.9.7'
         }
-        runner = HttpRunner(failfast=True).run(testcase_file_path, mapping=variables_mapping)
+        self.runner.run(testcase_file_path, mapping=variables_mapping)
-        summary = runner.summary
+        summary = self.runner.summary
         self.assertTrue(summary["success"])
         self.assertIn("token", summary["details"][0]["in_out"]["out"])
         # TODO: add
@@ -337,8 +334,8 @@ class TestHttpRunner(ApiServerUnittest):
     def test_run_testcase_with_parameters(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/data/demo_parameters.yml')
-        runner = HttpRunner().run(testcase_file_path)
+        self.runner.run(testcase_file_path)
-        summary = runner.summary
+        summary = self.runner.summary
         # TODO: add parameterize
         # self.assertEqual(
         #     summary["details"][0]["in_out"]["in"]["user_agent"],
@@ -364,8 +361,7 @@ class TestHttpRunner(ApiServerUnittest):
             os.getcwd(), 'tests/data/demo_parameters.yml')
         tests_mapping = loader.load_tests(testcase_file_path)
         parser.parse_tests(tests_mapping)
-        runner = HttpRunner()
+        test_suite = self.runner._add_tests(tests_mapping)
-        test_suite = runner._add_tests(tests_mapping)
 
         self.assertEqual(
             test_suite._tests[0].tests[0]['name'],
@@ -396,8 +392,8 @@ class TestHttpRunner(ApiServerUnittest):
     def test_validate_response_content(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/httpbin/basic.yml')
-        runner = HttpRunner().run(testcase_file_path)
+        self.runner.run(testcase_file_path)
-        self.assertTrue(runner.summary["success"])
+        self.assertTrue(self.runner.summary["success"])
 
 
 class TestApi(ApiServerUnittest):