From 95db6f22dd64bb4b66f9706b8c259a41e3e95f18 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 19 Jul 2018 15:15:09 +0800 Subject: [PATCH 01/10] refactor report: split test suite to separate tables --- httprunner/__about__.py | 2 +- httprunner/cli.py | 4 +- httprunner/context.py | 2 +- httprunner/report.py | 12 +- httprunner/task.py | 140 ++++++++++-------- .../templates/default_report_template.html | 80 ++++++---- httprunner/testcase.py | 3 - tests/test_httprunner.py | 10 +- tests/test_runner.py | 12 +- tests/test_task.py | 9 +- tests/test_testcase.py | 5 +- 11 files changed, 158 insertions(+), 121 deletions(-) diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 40f1461a..4741e330 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.4.8' +__version__ = '1.5.0' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/cli.py b/httprunner/cli.py index aeb74c2b..a9192021 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -88,7 +88,9 @@ def main_hrun(): ) summary = runner.summary - print_output(summary["output"]) + for suite_summary in summary["details"]: + print_output(suite_summary["output"]) + return 0 if summary["success"] else 1 def main_locust(): diff --git a/httprunner/context.py b/httprunner/context.py index 9059242c..77170155 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -221,7 +221,7 @@ class Context(object): def do_validation(self, validator_dict): """ validate with functions """ - # TODO: move comparator uniform to init_task_suite + # TODO: move comparator uniform to init_test_suites comparator = utils.get_uniform_comparator(validator_dict["comparator"]) validate_func = self.testcase_parser.get_bind_function(comparator) diff --git a/httprunner/report.py b/httprunner/report.py index 714eee1b..7b5220db 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -38,8 +38,7 @@ def get_summary(result): 'skipped': len(result.skipped), 'expectedFailures': len(result.expectedFailures), 'unexpectedSuccesses': len(result.unexpectedSuccesses) - }, - "platform": get_platform() + } } summary["stat"]["successes"] = summary["stat"]["testsRun"] \ - summary["stat"]["failures"] \ @@ -90,10 +89,11 @@ def render_html_report(summary, html_report_name=None, html_report_template=None if not os.path.isdir(report_dir_path): os.makedirs(report_dir_path) - for record in summary.get("records"): - meta_data = record['meta_data'] - stringify_body(meta_data, 'request') - stringify_body(meta_data, 'response') + for suite_summary in summary["details"]: + for record in suite_summary.get("records"): + meta_data = record['meta_data'] + stringify_body(meta_data, 'request') + stringify_body(meta_data, 'response') with io.open(html_report_template, "r", encoding='utf-8') as fp_r: template_content = fp_r.read() diff --git a/httprunner/task.py b/httprunner/task.py index 3448045c..1f02fde2 100644 --- a/httprunner/task.py +++ b/httprunner/task.py @@ -6,7 +6,8 @@ import unittest from httprunner import exception, logger, runner, testcase, utils from httprunner.compat import is_py3 -from httprunner.report import HtmlTestResult, get_summary, render_html_report +from httprunner.report import (HtmlTestResult, get_platform, get_summary, + render_html_report) from httprunner.testcase import TestcaseLoader from httprunner.utils import load_dot_env_file @@ -29,6 +30,7 @@ class TestCase(unittest.TestCase): self.meta_data = self.test_runner.http_client_session.meta_data self.test_runner.http_client_session.init_meta_data() + class TestSuite(unittest.TestSuite): """ create test suite with a testset, it may include one or several testcases. each suite should initialize a separate Runner() with testset config. @@ -64,12 +66,12 @@ class TestSuite(unittest.TestSuite): super(TestSuite, self).__init__() self.test_runner_list = [] - config_dict = testset.get("config", {}) - self.output_variables_list = config_dict.get("output", []) - self.testset_file_path = config_dict.get("path") - config_dict_parameters = config_dict.get("parameters", []) + self.config = testset.get("config", {}) + self.output_variables_list = self.config.get("output", []) + self.testset_file_path = self.config.get("path") + config_dict_parameters = self.config.get("parameters", []) - config_dict_variables = config_dict.get("variables", []) + config_dict_variables = self.config.get("variables", []) variables_mapping = variables_mapping or {} config_dict_variables = utils.override_variables_binds(config_dict_variables, variables_mapping) @@ -82,8 +84,8 @@ class TestSuite(unittest.TestSuite): for config_variables in config_parametered_variables_list: # config level - config_dict["variables"] = config_variables - test_runner = runner.Runner(config_dict, http_client_session) + self.config["variables"] = config_variables + test_runner = runner.Runner(self.config, http_client_session) for testcase_dict in testcases: testcase_dict = copy.copy(testcase_dict) @@ -148,55 +150,30 @@ class TestSuite(unittest.TestSuite): if not out: continue - outputs.append({"in": variables, "out": out}) + in_out = {"in": variables, "out": out} + if in_out not in outputs: + outputs.append(in_out) return outputs -class TaskSuite(unittest.TestSuite): - """ create task suite with specified testcase path. - each task suite may include one or several test suite. - """ - def __init__(self, testsets, mapping=None, http_client_session=None): - """ - @params - testsets (dict/list): testset or list of testset - testset_dict - or - [ - testset_dict_1, - testset_dict_2, - { - "name": "desc1", - "config": {}, - "api": {}, - "testcases": [testcase11, testcase12] - } - ] - mapping (dict): - passed in variables mapping, it will override variables in config block - """ - super(TaskSuite, self).__init__() - mapping = mapping or {} - if not testsets: - raise exception.TestcaseNotFound - - if isinstance(testsets, dict): - testsets = [testsets] - - self.suite_list = [] - for testset in testsets: - suite = TestSuite(testset, mapping, http_client_session) - self.addTest(suite) - self.suite_list.append(suite) - - @property - def tasks(self): - return self.suite_list - - -def init_task_suite(path_or_testsets, mapping=None, http_client_session=None): - """ initialize task suite +def init_test_suites(path_or_testsets, mapping=None, http_client_session=None): + """ initialize TestSuite list with testset path or testset dict + @params + testsets (dict/list): testset or list of testset + testset_dict + or + [ + testset_dict_1, + testset_dict_2, + { + "config": {}, + "api": {}, + "testcases": [testcase11, testcase12] + } + ] + mapping (dict): + passed in variables mapping, it will override variables in config block """ if not testcase.is_testsets(path_or_testsets): TestcaseLoader.load_test_dependencies() @@ -206,7 +183,19 @@ def init_task_suite(path_or_testsets, mapping=None, http_client_session=None): # TODO: move comparator uniform here mapping = mapping or {} - return TaskSuite(testsets, mapping, http_client_session) + + if not testsets: + raise exception.TestcaseNotFound + + if isinstance(testsets, dict): + testsets = [testsets] + + test_suite_list = [] + for testset in testsets: + test_suite = TestSuite(testset, mapping, http_client_session) + test_suite_list.append(test_suite) + + return test_suite_list class HttpRunner(object): @@ -242,19 +231,42 @@ class HttpRunner(object): if mapping specified, it will override variables in config block """ try: - task_suite = init_task_suite(path_or_testsets, mapping) + test_suite_list = init_test_suites(path_or_testsets, mapping) except exception.TestcaseNotFound: logger.log_error("Testcases not found in {}".format(path_or_testsets)) sys.exit(1) - result = self.runner.run(task_suite) - self.summary = get_summary(result) + self.summary = { + "success": True, + "stat": {}, + "time": {}, + "platform": get_platform(), + "details": [] + } - output = [] - for task in task_suite.tasks: - output.extend(task.output) + def accumulate_stat(origin_stat, new_stat): + """ accumulate new_stat to origin_stat + """ + for key in new_stat: + if key not in origin_stat: + origin_stat[key] = new_stat[key] + elif key == "start_at": + # start datetime + origin_stat[key] = min(origin_stat[key], new_stat[key]) + else: + origin_stat[key] += new_stat[key] + + for test_suite in test_suite_list: + result = self.runner.run(test_suite) + test_suite_summary = get_summary(result) + self.summary["success"] &= test_suite_summary["success"] + test_suite_summary["name"] = test_suite.config.get("name") + test_suite_summary["base_url"] = test_suite.config.get("request", {}).get("base_url", "") + test_suite_summary["output"] = test_suite.output + accumulate_stat(self.summary["stat"], test_suite_summary["stat"]) + accumulate_stat(self.summary["time"], test_suite_summary["time"]) + self.summary["details"].append(test_suite_summary) - self.summary["output"] = output return self def gen_html_report(self, html_report_name=None, html_report_template=None): @@ -274,11 +286,11 @@ class HttpRunner(object): class LocustTask(object): def __init__(self, path_or_testsets, locust_client, mapping=None): - self.task_suite = init_task_suite(path_or_testsets, mapping, locust_client) + self.test_suite_list = init_test_suites(path_or_testsets, mapping, locust_client) def run(self): - for suite in self.task_suite: - for test in suite: + for test_suite in self.test_suite_list: + for test in test_suite: try: test.runTest() except exception.MyBaseError as ex: diff --git a/httprunner/templates/default_report_template.html b/httprunner/templates/default_report_template.html index ccae9cad..9f9f8b38 100644 --- a/httprunner/templates/default_report_template.html +++ b/httprunner/templates/default_report_template.html @@ -9,7 +9,7 @@ margin: 0 auto; width: 960px; } - #summary, #details { + #summary { width: 960px; margin-bottom: 20px; } @@ -22,30 +22,34 @@ text-align: center; padding: 4px 8px; } - #details th { + .details { + width: 960px; + margin-bottom: 20px; + } + .details th { background-color: skyblue; padding: 5px 12px; } - #details td { + .details td { background-color: lightblue; padding: 5px 12px; } - #details .detail { + .details .detail { background-color: lightgrey; font-size: smaller; padding: 5px 10px; text-align: center; } - #details .success { + .details .success { background-color: greenyellow; } - #details .error { + .details .error { background-color: red; } - #details .failure { + .details .failure { background-color: salmon; } - #details .skipped { + .details .skipped { background-color: gray; } @@ -170,25 +174,42 @@

Details

- - - - - - - - {% for record in records %} - + + {% for test_suite_summary in details %} + {% set suite_index = loop.index %} +

{{test_suite_summary.name}}

+
StatusNameResponse TimeDetail
+ + + + + + + + + + + + + + + + + + + {% for record in test_suite_summary.records %} + {% set record_index = "{}_{}".format(suite_index, loop.index) %} + - + + {% endfor %} -
base_url{{test_suite_summary.base_url}}
TOTAL: {{test_suite_summary.stat.testsRun}}SUCCESS: {{test_suite_summary.stat.successes}}FAILED: {{test_suite_summary.stat.failures}}ERROR: {{test_suite_summary.stat.errors}}SKIPPED: {{test_suite_summary.stat.skipped}}
StatusNameResponse TimeDetail
{{record.status}} - {{record.name}}{{ record.meta_data["response_time_ms"] }} ms{{record.name}}{{ record.meta_data.response_time_ms }} ms - log -
+ + {% endfor %} \ No newline at end of file diff --git a/httprunner/testcase.py b/httprunner/testcase.py index 46ec1c19..e25eda8b 100644 --- a/httprunner/testcase.py +++ b/httprunner/testcase.py @@ -206,13 +206,11 @@ class TestcaseLoader(object): ] @return testset dict { - "name": "desc1", "config": {}, "testcases": [testcase11, testcase12] } """ testset = { - "name": "", "config": { "path": file_path }, @@ -228,7 +226,6 @@ class TestcaseLoader(object): if key == "config": testset["config"].update(test_block) - testset["name"] = test_block.get("name", "") elif key == "test": if "api" in test_block: diff --git a/tests/test_httprunner.py b/tests/test_httprunner.py index 4d940e3f..dae01d8a 100644 --- a/tests/test_httprunner.py +++ b/tests/test_httprunner.py @@ -90,7 +90,8 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 2) - self.assertIn("records", summary) + self.assertIn("details", summary) + self.assertIn("records", summary["details"][0]) def test_run_testset(self): testsets = self.testset @@ -98,7 +99,7 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 2) - self.assertIn("records", summary) + self.assertIn("records", summary["details"][0]) def test_run_yaml_upload(self): testset_path = "tests/httpbin/upload.yml" @@ -106,7 +107,8 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 1) - self.assertIn("records", summary) + self.assertIn("details", summary) + self.assertIn("records", summary["details"][0]) def test_run_post_data(self): testsets = [ @@ -135,7 +137,7 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 1) - self.assertEqual(summary["records"][0]["meta_data"]["response_body"]["data"], "abc") + self.assertEqual(summary["details"][0]["records"][0]["meta_data"]["response_body"]["data"], "abc") def test_html_report_repsonse_image(self): testset_path = "tests/httpbin/load_image.yml" diff --git a/tests/test_runner.py b/tests/test_runner.py index 24874538..460f90c6 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -282,8 +282,9 @@ class TestRunner(ApiServerUnittest): runner = HttpRunner().run(testcase_file_path) summary = runner.summary self.assertTrue(summary["success"]) - self.assertIn("token", summary["output"][0]["out"]) - self.assertEqual(len(summary["output"]), 13) + self.assertIn("token", summary["details"][0]["output"][0]["out"]) + #TODO: fix + self.assertEqual(len(summary["details"][0]["output"]), 3) def test_run_testset_with_variables_mapping(self): testcase_file_path = os.path.join( @@ -294,8 +295,9 @@ class TestRunner(ApiServerUnittest): runner = HttpRunner().run(testcase_file_path, mapping=variables_mapping) summary = runner.summary self.assertTrue(summary["success"]) - self.assertIn("token", summary["output"][0]["out"]) - self.assertEqual(len(summary["output"]), 13) + self.assertIn("token", summary["details"][0]["output"][0]["out"]) + #TODO: fix + self.assertEqual(len(summary["details"][0]["output"]), 3) def test_run_testcase_with_empty_header(self): testcase_file_path = os.path.join( @@ -328,5 +330,5 @@ class TestRunner(ApiServerUnittest): runner = HttpRunner().run(testcase_file_path) summary = runner.summary self.assertTrue(summary["success"]) - self.assertEqual(len(summary["output"]), 3 * 2 * 2) + self.assertEqual(len(summary["details"][0]["output"]), 3 * 2 * 2) self.assertEqual(summary["stat"]["testsRun"], 3 * 2 * 2) diff --git a/tests/test_task.py b/tests/test_task.py index 22000023..7e49d6a4 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -73,8 +73,9 @@ class TestTask(ApiServerUnittest): ] } ] - task_suite = task.TaskSuite(testsets) + test_suite_list = task.init_test_suites(testsets) + self.assertEqual(len(test_suite_list), 1) + task_suite = test_suite_list[0] self.assertEqual(task_suite.countTestCases(), 2) - for suite in task_suite: - for testcase in suite: - self.assertIsInstance(testcase, task.TestCase) + for testcase in task_suite: + self.assertIsInstance(testcase, task.TestCase) diff --git a/tests/test_testcase.py b/tests/test_testcase.py index d25f4e89..2e4af88c 100644 --- a/tests/test_testcase.py +++ b/tests/test_testcase.py @@ -34,7 +34,6 @@ class TestTestcaseLoader(unittest.TestCase): def test_load_test_file_suite(self): TestcaseLoader.load_api_file("tests/api/basic.yml") testset = TestcaseLoader.load_test_file("tests/suite/create_and_get.yml") - self.assertEqual(testset["name"], "create user and check result.") self.assertEqual(testset["config"]["name"], "create user and check result.") self.assertEqual(len(testset["testcases"]), 3) self.assertEqual(testset["testcases"][0]["name"], "make sure user $uid does not exist") @@ -43,7 +42,7 @@ class TestTestcaseLoader(unittest.TestCase): def test_load_test_file_testcase(self): TestcaseLoader.load_test_dependencies() testset = TestcaseLoader.load_test_file("tests/testcases/smoketest.yml") - self.assertEqual(testset["name"], "smoketest") + self.assertEqual(testset["config"]["name"], "smoketest") self.assertEqual(testset["config"]["path"], "tests/testcases/smoketest.yml") self.assertIn("device_sn", testset["config"]["variables"][0]) self.assertEqual(len(testset["testcases"]), 8) @@ -76,7 +75,7 @@ class TestTestcaseLoader(unittest.TestCase): def test_get_test_definition_suite(self): TestcaseLoader.load_test_dependencies() api_def = TestcaseLoader._get_test_definition("create_and_check", "suite") - self.assertEqual(api_def["name"], "create user and check result.") + self.assertEqual(api_def["config"]["name"], "create user and check result.") with self.assertRaises(SuiteNotFound): TestcaseLoader._get_test_definition("create_and_check_XXX", "suite") From baaca1290fcced4be508b55105218fe192b37731 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 19 Jul 2018 15:27:00 +0800 Subject: [PATCH 02/10] call print_output after each testset --- httprunner/cli.py | 6 +----- httprunner/task.py | 6 +++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/httprunner/cli.py b/httprunner/cli.py index a9192021..fdabd9a7 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -11,8 +11,7 @@ from httprunner.__about__ import __description__, __version__ from httprunner.compat import is_py2 from httprunner.task import HttpRunner from httprunner.utils import (create_scaffold, get_python2_retire_msg, - prettify_json_file, print_output, - validate_json_file) + prettify_json_file, validate_json_file) def main_hrun(): @@ -88,9 +87,6 @@ def main_hrun(): ) summary = runner.summary - for suite_summary in summary["details"]: - print_output(suite_summary["output"]) - return 0 if summary["success"] else 1 def main_locust(): diff --git a/httprunner/task.py b/httprunner/task.py index 1f02fde2..76f28fa0 100644 --- a/httprunner/task.py +++ b/httprunner/task.py @@ -9,7 +9,7 @@ from httprunner.compat import is_py3 from httprunner.report import (HtmlTestResult, get_platform, get_summary, render_html_report) from httprunner.testcase import TestcaseLoader -from httprunner.utils import load_dot_env_file +from httprunner.utils import load_dot_env_file, print_output class TestCase(unittest.TestCase): @@ -259,12 +259,16 @@ class HttpRunner(object): for test_suite in test_suite_list: result = self.runner.run(test_suite) test_suite_summary = get_summary(result) + self.summary["success"] &= test_suite_summary["success"] test_suite_summary["name"] = test_suite.config.get("name") test_suite_summary["base_url"] = test_suite.config.get("request", {}).get("base_url", "") test_suite_summary["output"] = test_suite.output + print_output(test_suite_summary["output"]) + accumulate_stat(self.summary["stat"], test_suite_summary["stat"]) accumulate_stat(self.summary["time"], test_suite_summary["time"]) + self.summary["details"].append(test_suite_summary) return self From 4ab3666b52ef8a631b9b5aef52cc94b602438a1d Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 19 Jul 2018 23:56:24 +0800 Subject: [PATCH 03/10] display validators in report log --- httprunner/__about__.py | 2 +- httprunner/context.py | 22 +++++++---- httprunner/runner.py | 6 ++- httprunner/task.py | 1 + .../templates/default_report_template.html | 37 +++++++++++++++++++ tests/test_context.py | 24 ++++++++---- 6 files changed, 73 insertions(+), 19 deletions(-) diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 4741e330..80ebd1fb 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.5.0' +__version__ = '1.5.1' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/context.py b/httprunner/context.py index 77170155..9decb7ca 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -216,6 +216,7 @@ class Context(object): # 2, actual value, e.g. 200 expect_value = self.eval_content(validator["expect"]) validator["expect"] = expect_value + validator["check_result"] = "unchecked" return validator def do_validation(self, validator_dict): @@ -237,6 +238,7 @@ class Context(object): raise exception.ParamsError("Null value can only be compared with comparator: eq/equals/==") try: + validator_dict["check_result"] = "passed" validate_func(validator_dict["check_value"], validator_dict["expect"]) except (AssertionError, TypeError): err_msg = "\n" + "\n".join([ @@ -245,18 +247,22 @@ class Context(object): "\tcomparator: %s;" % comparator, "\texpected value: %s (%s)." % (expect_value, type(expect_value).__name__) ]) + validator_dict["check_result"] = "failed" raise exception.ValidationError(err_msg) - def validate(self, validators, resp_obj): - """ check validators with the context variable mapping. - @param (list) validators - @param (object) resp_obj + def eval_validators(self, validators, resp_obj): + """ evaluate validators with context variable mapping. """ - for validator in validators: - validator_dict = self.eval_check_item( + return [ + self.eval_check_item( testcase.parse_validator(validator), resp_obj ) - self.do_validation(validator_dict) + for validator in validators + ] - return True + def validate(self, validators): + """ make validations + """ + for validator_dict in validators: + self.do_validation(validator_dict) diff --git a/httprunner/runner.py b/httprunner/runner.py index 00919d12..fbf8031f 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -11,6 +11,7 @@ class Runner(object): def __init__(self, config_dict=None, http_client_session=None): self.http_client_session = http_client_session + self.evaluated_validators = [] self.context = Context() config_dict = config_dict or {} @@ -178,11 +179,12 @@ class Runner(object): extractors = testcase_dict.get("extract", []) or testcase_dict.get("extractors", []) extracted_variables_mapping = resp_obj.extract_response(extractors) self.context.bind_extracted_variables(extracted_variables_mapping) - + # validate validators = testcase_dict.get("validate", []) or testcase_dict.get("validators", []) try: - self.context.validate(validators, resp_obj) + self.evaluated_validators = self.context.eval_validators(validators, resp_obj) + self.context.validate(self.evaluated_validators) except (exception.ParamsError, exception.ResponseError, \ exception.ValidationError, exception.ParseResponseError): # log request diff --git a/httprunner/task.py b/httprunner/task.py index 76f28fa0..62bce835 100644 --- a/httprunner/task.py +++ b/httprunner/task.py @@ -28,6 +28,7 @@ class TestCase(unittest.TestCase): finally: if hasattr(self.test_runner.http_client_session, "meta_data"): self.meta_data = self.test_runner.http_client_session.meta_data + self.meta_data["validators"] = self.test_runner.evaluated_validators self.test_runner.http_client_session.init_meta_data() diff --git a/httprunner/templates/default_report_template.html b/httprunner/templates/default_report_template.html index 9f9f8b38..a2ddcd22 100644 --- a/httprunner/templates/default_report_template.html +++ b/httprunner/templates/default_report_template.html @@ -30,6 +30,15 @@ background-color: skyblue; padding: 5px 12px; } + .details tr .passed { + background-color: lightgreen; + } + .details tr .failed { + background-color: red; + } + .details tr .unchecked { + background-color: gray; + } .details td { background-color: lightblue; padding: 5px 12px; @@ -277,6 +286,34 @@ +

Validators:

+
+ + + + + + + + {% for validator in record.meta_data.validators %} + + {% if validator.check_result == "passed" %} + + + + + + {% endfor %} +
checkcomparatorexpect valueactual value
+ {% elif validator.check_result == "failed" %} + + {% elif validator.check_result == "unchecked" %} + + {% endif %} + {{validator.check}} + {{validator.comparator}}{{validator.expect}}{{validator.check_value}}
+
+

Statistics:

diff --git a/tests/test_context.py b/tests/test_context.py index 647390bc..746c79e3 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -262,8 +262,7 @@ class VariableBindsUnittest(ApiServerUnittest): validators = [ {"eq": ["$resp_status_code", 201]}, {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, - {"check": "$resp_body_success", "comparator": "eq", "expect": True}, - {"check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False} + {"check": "$resp_body_success", "comparator": "eq", "expect": True} ] variables = [ {"resp_status_code": 200}, @@ -272,8 +271,15 @@ class VariableBindsUnittest(ApiServerUnittest): self.context.bind_variables(variables) with self.assertRaises(exception.ValidationError): - self.context.validate(validators, resp_obj) + evaluated_validators = self.context.eval_validators(validators, resp_obj) + self.context.validate(evaluated_validators) + validators = [ + {"eq": ["$resp_status_code", 201]}, + {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, + {"check": "$resp_body_success", "comparator": "eq", "expect": True}, + {"check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False} + ] variables = [ {"resp_status_code": 201}, {"resp_body_success": True} @@ -285,7 +291,8 @@ class VariableBindsUnittest(ApiServerUnittest): } self.context.bind_functions(functions) - self.assertTrue(self.context.validate(validators, resp_obj)) + evaluated_validators = self.context.eval_validators(validators, resp_obj) + self.context.validate(evaluated_validators) def test_validate_exception(self): url = "http://127.0.0.1:5000/" @@ -295,14 +302,14 @@ class VariableBindsUnittest(ApiServerUnittest): # expected value missed in validators validators = [ {"eq": ["$resp_status_code", 201]}, - {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, - {"check": "$resp_body_success", "comparator": "eq", "expect": True} + {"check": "$resp_status_code", "comparator": "eq", "expect": 201} ] variables = [] self.context.bind_variables(variables) with self.assertRaises(exception.ParamsError): - self.context.validate(validators, resp_obj) + evaluated_validators = self.context.eval_validators(validators, resp_obj) + self.context.validate(evaluated_validators) # expected value missed in variables mapping variables = [ @@ -311,4 +318,5 @@ class VariableBindsUnittest(ApiServerUnittest): self.context.bind_variables(variables) with self.assertRaises(exception.ValidationError): - self.context.validate(validators, resp_obj) + evaluated_validators = self.context.eval_validators(validators, resp_obj) + self.context.validate(evaluated_validators) From 97e884de616e714c7dd1adfc9dc40de6410e3e6f Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 20 Jul 2018 11:06:34 +0800 Subject: [PATCH 04/10] display parameters and output in report --- httprunner/__about__.py | 2 +- httprunner/task.py | 5 +++- .../templates/default_report_template.html | 29 +++++++++++++++++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 80ebd1fb..792bf03a 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.5.1' +__version__ = '1.5.2' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/task.py b/httprunner/task.py index 62bce835..965132c9 100644 --- a/httprunner/task.py +++ b/httprunner/task.py @@ -151,7 +151,10 @@ class TestSuite(unittest.TestSuite): if not out: continue - in_out = {"in": variables, "out": out} + in_out = { + "in": variables or {}, + "out": out + } if in_out not in outputs: outputs.append(in_out) diff --git a/httprunner/templates/default_report_template.html b/httprunner/templates/default_report_template.html index a2ddcd22..e72587b0 100644 --- a/httprunner/templates/default_report_template.html +++ b/httprunner/templates/default_report_template.html @@ -187,10 +187,35 @@ {% for test_suite_summary in details %} {% set suite_index = loop.index %}

{{test_suite_summary.name}}

-
+
- + + From d35743ab940e43ad3b1a59188d312fbfdc868e96 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 20 Jul 2018 11:39:15 +0800 Subject: [PATCH 05/10] convert OrderedDict to dict --- httprunner/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httprunner/task.py b/httprunner/task.py index 965132c9..c65a7809 100644 --- a/httprunner/task.py +++ b/httprunner/task.py @@ -152,7 +152,7 @@ class TestSuite(unittest.TestSuite): continue in_out = { - "in": variables or {}, + "in": dict(variables), "out": out } if in_out not in outputs: From b48dbc27b8b1bc980ade81c0ba0c0af8a3ebf273 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 20 Jul 2018 18:15:22 +0800 Subject: [PATCH 06/10] convert datetime object to string in summary data structure --- httprunner/__about__.py | 2 +- httprunner/report.py | 9 +++++---- httprunner/templates/default_report_template.html | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 792bf03a..a5a8532d 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.5.2' +__version__ = '1.5.3' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/report.py b/httprunner/report.py index 7b5220db..23b2bd64 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -49,7 +49,7 @@ def get_summary(result): if getattr(result, "records", None): summary["time"] = { - 'start_at': datetime.fromtimestamp(result.start_at), + 'start_at': result.start_at, 'duration': result.duration } summary["records"] = result.records @@ -77,14 +77,15 @@ def render_html_report(summary, html_report_name=None, html_report_template=None logger.log_debug("render data: {}".format(summary)) report_dir_path = os.path.join(os.getcwd(), "reports") - start_datetime = summary["time"]["start_at"].strftime('%Y-%m-%d-%H-%M-%S') + start_at_timestamp = int(summary["time"]["start_at"]) + summary["time"]["start_datetime"] = datetime.fromtimestamp(start_at_timestamp).strftime('%Y-%m-%d %H:%M:%S') if html_report_name: summary["html_report_name"] = html_report_name report_dir_path = os.path.join(report_dir_path, html_report_name) - html_report_name += "-{}.html".format(start_datetime) + html_report_name += "-{}.html".format(start_at_timestamp) else: summary["html_report_name"] = "" - html_report_name = "{}.html".format(start_datetime) + html_report_name = "{}.html".format(start_at_timestamp) if not os.path.isdir(report_dir_path): os.makedirs(report_dir_path) diff --git a/httprunner/templates/default_report_template.html b/httprunner/templates/default_report_template.html index e72587b0..06941c1a 100644 --- a/httprunner/templates/default_report_template.html +++ b/httprunner/templates/default_report_template.html @@ -150,7 +150,7 @@ - + From c7af7b3934d4845b6b26b0067de70859c3423b34 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 21 Jul 2018 11:40:44 +0800 Subject: [PATCH 07/10] record more info about request and response, and display in report --- httprunner/__about__.py | 2 +- httprunner/client.py | 90 +++++++++------- httprunner/report.py | 2 +- .../templates/default_report_template.html | 101 ++++++++---------- tests/test_httprunner.py | 2 +- 5 files changed, 100 insertions(+), 97 deletions(-) diff --git a/httprunner/__about__.py b/httprunner/__about__.py index a5a8532d..fc9fa35f 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.5.3' +__version__ = '1.5.4' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/client.py b/httprunner/client.py index 097fdd57..acfb3d90 100644 --- a/httprunner/client.py +++ b/httprunner/client.py @@ -55,17 +55,20 @@ class HttpSession(requests.Session): """ initialize meta_data, it will store detail data of request and response """ self.meta_data = { - "url": "N/A", - "method": "N/A", - "request_time": "N/A", - "request_headers": {}, - "request_body": "N/A", - "status_code": "N/A", - "response_headers": {}, - "response_body": "N/A", - "content_size": "N/A", - "response_time_ms": "N/A", - "elapsed_ms": "N/A" + "request": { + "url": "N/A", + "method": "N/A", + "headers": {}, + "start_timestamp": None + }, + "response": { + "status_code": "N/A", + "headers": {}, + "content_size": "N/A", + "response_time_ms": "N/A", + "elapsed_ms": "N/A", + "content": None + } } def request(self, method, url, name=None, **kwargs): @@ -107,10 +110,17 @@ class HttpSession(requests.Session): :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. """ + def log_print(request_response): + msg = "\n================== {} details ==================\n".format(request_response) + for key, value in self.meta_data[request_response].items(): + msg += "{:<16} : {}\n".format(key, value) + logger.log_debug(msg) + # record original request info - self.meta_data["method"] = method - self.meta_data["url"] = url - self.meta_data["request_time"] = time.time() + self.meta_data["request"]["method"] = method + self.meta_data["request"]["url"] = url + self.meta_data["request"].update(kwargs) + self.meta_data["request"]["start_timestamp"] = time.time() # prepend url with hostname unless it's already an absolute URL url = self._build_url(url) @@ -119,35 +129,43 @@ class HttpSession(requests.Session): response = self._send_request_safe_mode(method, url, **kwargs) # record the consumed time - self.meta_data["response_time_ms"] = round((time.time() - self.meta_data["request_time"]) * 1000, 2) - self.meta_data["elapsed_ms"] = response.elapsed.microseconds / 1000.0 + self.meta_data["response"]["response_time_ms"] = \ + round((time.time() - self.meta_data["request"]["start_timestamp"]) * 1000, 2) + self.meta_data["response"]["elapsed_ms"] = response.elapsed.microseconds / 1000.0 # record actual request info - self.meta_data["url"] = (response.history and response.history[0] or response).request.url - self.meta_data["request_headers"] = response.request.headers - self.meta_data["request_body"] = response.request.body + self.meta_data["request"]["url"] = (response.history and response.history[0] or response).request.url + self.meta_data["request"]["headers"] = response.request.headers + self.meta_data["request"]["body"] = response.request.body + + # log request details in debug mode + log_print("request") # record response info - self.meta_data["status_code"] = response.status_code - self.meta_data["response_headers"] = response.headers - try: - self.meta_data["response_body"] = response.json() - except ValueError: - self.meta_data["response_body"] = response.content + self.meta_data["response"]["ok"] = response.ok + self.meta_data["response"]["url"] = response.url + self.meta_data["response"]["status_code"] = response.status_code + self.meta_data["response"]["reason"] = response.reason + self.meta_data["response"]["headers"] = response.headers + self.meta_data["response"]["cookies"] = response.cookies or {} + self.meta_data["response"]["encoding"] = response.encoding + self.meta_data["response"]["content"] = response.content + self.meta_data["response"]["text"] = response.text - # log response details in debug mode - msg = "response details:\n" - msg += "> status_code: {}\n".format(self.meta_data["status_code"]) - msg += "> headers: {}\n".format(self.meta_data["response_headers"]) - msg += "> body: {}".format(self.meta_data["response_body"]) - logger.log_debug(msg) + try: + self.meta_data["response"]["json"] = response.json() + except ValueError: + self.meta_data["response"]["json"] = None # get the length of the content, but if the argument stream is set to True, we take # the size from the content-length header, in order to not trigger fetching of the body if kwargs.get("stream", False): - self.meta_data["content_size"] = int(self.meta_data["response_headers"].get("content-length") or 0) + self.meta_data["response"]["content_size"] = int(self.meta_data["response"]["headers"].get("content-length") or 0) else: - self.meta_data["content_size"] = len(response.content or "") + self.meta_data["response"]["content_size"] = len(response.content or "") + + # log response details in debug mode + log_print("response") try: response.raise_for_status() @@ -156,9 +174,9 @@ class HttpSession(requests.Session): else: logger.log_info( """status_code: {}, response_time(ms): {} ms, response_length: {} bytes""".format( - self.meta_data["status_code"], - self.meta_data["response_time_ms"], - self.meta_data["content_size"] + self.meta_data["response"]["status_code"], + self.meta_data["response"]["response_time_ms"], + self.meta_data["response"]["content_size"] ) ) diff --git a/httprunner/report.py b/httprunner/report.py index 23b2bd64..7141bffe 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -121,7 +121,7 @@ def stringify_body(meta_data, request_or_response): resp_content_type = headers.get("Content-Type", "") try: if "image" in resp_content_type: - meta_data["response_data_type"] = "image" + meta_data["response"]["data_type"] = "image" body = "data:{};base64,{}".format( resp_content_type, b64encode(body).decode('utf-8') diff --git a/httprunner/templates/default_report_template.html b/httprunner/templates/default_report_template.html index 06941c1a..ec11a2b0 100644 --- a/httprunner/templates/default_report_template.html +++ b/httprunner/templates/default_report_template.html @@ -236,7 +236,7 @@ - + + + + + {% endfor %} +
base_url{{test_suite_summary.base_url}}{{test_suite_summary.base_url}} + parameters & output +
+
+

Parameters and Output

+ × +
+
+ + + + + + {% for in_out in test_suite_summary.output %} + + + + + {% endfor %} +
variablesoutput
{{in_out.in}}{{in_out.out}}
+
+
+
+
+
TOTAL: {{test_suite_summary.stat.testsRun}}
START AT{{time.start_at.strftime('%Y-%m-%d %H:%M:%S')}}{{time.start_datetime}}
DURATION
{{record.status}} {{record.name}}{{ record.meta_data.response_time_ms }} ms{{ record.meta_data.response.response_time_ms }} ms log @@ -249,66 +249,51 @@

Request:

- - - - - - - - - - - - - {% if record.meta_data.method in ["POST", "PUT"] %} - - - - - {% endif %} - + {% for key, value in record.meta_data.request.items() %} + + + + + {% endfor %}
url{{ record.meta_data.url }}
method{{ record.meta_data.method }}
headers - {% for key, value in record.meta_data.request_headers.items() %} -
- {{ key }}: {{ value }} -
- {% endfor %} -
body - 
{{ record.meta_data.request_body }}
-
{{key}} + {% if key == "headers" %} + {% for header_key, header_value in record.meta_data.request.headers.items() %} +
+ {{ header_key }}: {{ header_value }} +
+ {% endfor %} + {% else %} + {{value}} + {% endif %} +

Response:

- - - - - - - - - - - - -
status_code - {{ record.meta_data.status_code }} -
headers - {% for key, value in record.meta_data.response_headers.items() %} -
- {{ key }}: {{ value }} -
- {% endfor %} -
body - {% if record.meta_data.response_data_type == "image" %} - - {% else %} - 
{{ record.meta_data.response_body }}
- {% endif %} -
+ {% for key, value in record.meta_data.response.items() %} +
{{key}} + {% if key == "headers" %} + {% for header_key, header_value in record.meta_data.request.headers.items() %} +
+ {{ header_key }}: {{ header_value }} +
+ {% endfor %} + {% elif key == "data_type" %} + {% if value == "image" %} + + {% else %} + 
{{ record.meta_data.response.body }}
+ {% endif %} + {% else %} + {{value}} + {% endif %} +

Validators:

@@ -344,15 +329,15 @@ - + - + - +
content_size(bytes){{ record.meta_data.content_size }}{{ record.meta_data.response.content_size }}
response_time(ms){{ record.meta_data.response_time_ms }}{{ record.meta_data.response.response_time_ms }}
elapsed(ms){{ record.meta_data.elapsed_ms }}{{ record.meta_data.response.elapsed_ms }}
diff --git a/tests/test_httprunner.py b/tests/test_httprunner.py index dae01d8a..cadcae15 100644 --- a/tests/test_httprunner.py +++ b/tests/test_httprunner.py @@ -137,7 +137,7 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 1) - self.assertEqual(summary["details"][0]["records"][0]["meta_data"]["response_body"]["data"], "abc") + self.assertEqual(summary["details"][0]["records"][0]["meta_data"]["response"]["json"]["data"], "abc") def test_html_report_repsonse_image(self): testset_path = "tests/httpbin/load_image.yml" From c8cabedad82931093361ab68ac8f93bc1f66be1d Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 21 Jul 2018 12:25:23 +0800 Subject: [PATCH 08/10] convert CaseInsensitiveDict to dict --- httprunner/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/httprunner/client.py b/httprunner/client.py index acfb3d90..187daf30 100644 --- a/httprunner/client.py +++ b/httprunner/client.py @@ -135,7 +135,7 @@ class HttpSession(requests.Session): # record actual request info self.meta_data["request"]["url"] = (response.history and response.history[0] or response).request.url - self.meta_data["request"]["headers"] = response.request.headers + self.meta_data["request"]["headers"] = dict(response.request.headers) self.meta_data["request"]["body"] = response.request.body # log request details in debug mode @@ -146,7 +146,7 @@ class HttpSession(requests.Session): self.meta_data["response"]["url"] = response.url self.meta_data["response"]["status_code"] = response.status_code self.meta_data["response"]["reason"] = response.reason - self.meta_data["response"]["headers"] = response.headers + self.meta_data["response"]["headers"] = dict(response.headers) self.meta_data["response"]["cookies"] = response.cookies or {} self.meta_data["response"]["encoding"] = response.encoding self.meta_data["response"]["content"] = response.content From 1c8d38a2c8d58c24105ffba0d355122490eada82 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 21 Jul 2018 12:47:39 +0800 Subject: [PATCH 09/10] bugfix: stringfy data before generating report --- httprunner/__about__.py | 2 +- httprunner/client.py | 5 +- httprunner/report.py | 55 ++++++++++--------- .../templates/default_report_template.html | 2 +- 4 files changed, 35 insertions(+), 29 deletions(-) diff --git a/httprunner/__about__.py b/httprunner/__about__.py index fc9fa35f..995d5ee3 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.5.4' +__version__ = '1.5.5' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/client.py b/httprunner/client.py index 187daf30..9b2ecfd5 100644 --- a/httprunner/client.py +++ b/httprunner/client.py @@ -67,7 +67,9 @@ class HttpSession(requests.Session): "content_size": "N/A", "response_time_ms": "N/A", "elapsed_ms": "N/A", - "content": None + "encoding": None, + "content": None, + "content_type": "" } } @@ -151,6 +153,7 @@ class HttpSession(requests.Session): self.meta_data["response"]["encoding"] = response.encoding self.meta_data["response"]["content"] = response.content self.meta_data["response"]["text"] = response.text + self.meta_data["response"]["content_type"] = response.headers.get("Content-Type", "") try: self.meta_data["response"]["json"] = response.json() diff --git a/httprunner/report.py b/httprunner/report.py index 7141bffe..7cb6b172 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -93,8 +93,8 @@ def render_html_report(summary, html_report_name=None, html_report_template=None for suite_summary in summary["details"]: for record in suite_summary.get("records"): meta_data = record['meta_data'] - stringify_body(meta_data, 'request') - stringify_body(meta_data, 'response') + stringify_data(meta_data, 'request') + stringify_data(meta_data, 'response') with io.open(html_report_template, "r", encoding='utf-8') as fp_r: template_content = fp_r.read() @@ -107,36 +107,39 @@ def render_html_report(summary, html_report_name=None, html_report_template=None return report_path -def stringify_body(meta_data, request_or_response): - headers = meta_data.get('{}_headers'.format(request_or_response), {}) - body = meta_data.get('{}_body'.format(request_or_response)) +def stringify_data(meta_data, request_or_response): + headers = meta_data[request_or_response]["headers"] - if isinstance(body, CaseInsensitiveDict): - body = json.dumps(dict(body), ensure_ascii=False) + request_or_response_dict = meta_data[request_or_response] - elif isinstance(body, (dict, list)): - body = json.dumps(body, indent=2, ensure_ascii=False) + for key, value in request_or_response_dict.items(): - elif isinstance(body, bytes): - resp_content_type = headers.get("Content-Type", "") - try: - if "image" in resp_content_type: - meta_data["response"]["data_type"] = "image" - body = "data:{};base64,{}".format( - resp_content_type, - b64encode(body).decode('utf-8') - ) - else: - body = escape(body.decode("utf-8")) - except UnicodeDecodeError: - pass + if isinstance(value, list): + value = json.dumps(value, indent=2, ensure_ascii=False) - elif not isinstance(body, (basestring, numeric_types, Iterable)): - # class instance, e.g. MultipartEncoder() - body = repr(body) + elif isinstance(value, bytes): + try: + encoding = meta_data["response"].get("encoding") + if not encoding or encoding == "None": + encoding = "utf-8" - meta_data['{}_body'.format(request_or_response)] = body + content_type = meta_data["response"]["content_type"] + if "image" in content_type: + meta_data["response"]["content_type"] = "image" + value = "data:{};base64,{}".format( + content_type, + b64encode(value).decode(encoding) + ) + else: + value = escape(value.decode(encoding)) + except UnicodeDecodeError: + pass + elif not isinstance(value, (basestring, numeric_types, Iterable)): + # class instance, e.g. MultipartEncoder() + value = repr(value) + + meta_data[request_or_response][key] = value class HtmlTestResult(unittest.TextTestResult): """A html result class that can generate formatted html results. diff --git a/httprunner/templates/default_report_template.html b/httprunner/templates/default_report_template.html index ec11a2b0..4c7b4441 100644 --- a/httprunner/templates/default_report_template.html +++ b/httprunner/templates/default_report_template.html @@ -281,7 +281,7 @@ {{ header_key }}: {{ header_value }} {% endfor %} - {% elif key == "data_type" %} + {% elif key == "content_type" %} {% if value == "image" %} {% else %} From e2a46501fb7bc04b6ebf6e1efca8a25a7f41d3fd Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 23 Jul 2018 21:30:01 +0800 Subject: [PATCH 10/10] support validate with response time --- httprunner/__about__.py | 2 +- httprunner/response.py | 16 +++++++++++++--- tests/test_runner.py | 27 +++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 995d5ee3..9fc5c801 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.5.5' +__version__ = '1.5.6' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/response.py b/httprunner/response.py index d00a397c..9a2a3b89 100644 --- a/httprunner/response.py +++ b/httprunner/response.py @@ -82,6 +82,16 @@ class ResponseObject(object): err_msg += u"attribute: {}".format(sub_query) logger.log_error(err_msg) raise exception.ParamsError(err_msg) + elif top_query == "elapsed": + if sub_query in ["days", "seconds", "microseconds"]: + return getattr(self.elapsed, sub_query) + elif sub_query == "total_seconds": + return self.elapsed.total_seconds() + else: + err_msg = "{}: {} is not valid timedelta attribute.\n".format(field, sub_query) + err_msg += "elapsed only support attributes: days, seconds, microseconds, total_seconds.\n" + logger.log_error(err_msg) + raise exception.ParamsError(err_msg) try: top_query_content = getattr(self, top_query) @@ -96,11 +106,11 @@ class ResponseObject(object): # TODO: remove compatibility for content, text if isinstance(top_query_content, bytes): top_query_content = top_query_content.decode("utf-8") - + if isinstance(top_query_content, PreparedRequest): top_query_content = top_query_content.__dict__ - else: - top_query_content = json.loads(top_query_content) + else: + top_query_content = json.loads(top_query_content) except json.decoder.JSONDecodeError: err_msg = u"Failed to extract data with delimiter!\n" err_msg += u"response content: {}\n".format(self.content) diff --git a/tests/test_runner.py b/tests/test_runner.py index 460f90c6..390956eb 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -332,3 +332,30 @@ class TestRunner(ApiServerUnittest): self.assertTrue(summary["success"]) self.assertEqual(len(summary["details"][0]["output"]), 3 * 2 * 2) self.assertEqual(summary["stat"]["testsRun"], 3 * 2 * 2) + + def test_run_validate_elapsed(self): + test = { + "name": "get token", + "request": { + "url": "http://127.0.0.1:5000/api/get-token", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3", + "device_sn": "HZfFBh6tU59EdXJ", + "os_platform": "ios", + "app_version": "2.8.6" + }, + "json": { + "sign": "f1219719911caae89ccc301679857ebfda115ca2" + } + }, + "validate": [ + {"check": "status_code", "expect": 200}, + {"check": "elapsed.seconds", "comparator": "lt", "expect": 1}, + {"check": "elapsed.days", "comparator": "eq", "expect": 0}, + {"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000}, + {"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1} + ] + } + self.test_runner.run_test(test)