GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-09-04 23:16:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor report: split test suite to separate tables

Browse Source
This commit is contained in:
debugtalk
2018-07-19 15:15:09 +08:00
parent 1f2e0d312a
commit 95db6f22dd
11 changed files with 158 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
httprunner/__about__.py
+1 -1
View File
@@ -1,7 +1,7 @@
__title__ = 'HttpRunner' __title__ = 'HttpRunner'
__description__ = 'One-stop solution for HTTP(S) testing.' __description__ = 'One-stop solution for HTTP(S) testing.'
__url__ = 'https://github.com/HttpRunner/HttpRunner' __url__ = 'https://github.com/HttpRunner/HttpRunner'
__version__ = '1.4.8' __version__ = '1.5.0'
__author__ = 'debugtalk' __author__ = 'debugtalk'
__author_email__ = 'mail@debugtalk.com' __author_email__ = 'mail@debugtalk.com'
__license__ = 'MIT' __license__ = 'MIT'
httprunner/cli.py
+3 -1
View File
@@ -88,7 +88,9 @@ def main_hrun():
) )
summary = runner.summary summary = runner.summary
print_output(summary["output"]) for suite_summary in summary["details"]:
print_output(suite_summary["output"])
return 0 if summary["success"] else 1 return 0 if summary["success"] else 1
def main_locust(): def main_locust():
httprunner/context.py
+1 -1
View File
@@ -221,7 +221,7 @@ class Context(object):
def do_validation(self, validator_dict): def do_validation(self, validator_dict):
""" validate with functions """ validate with functions
""" """
# TODO: move comparator uniform to init_task_suite # TODO: move comparator uniform to init_test_suites
comparator = utils.get_uniform_comparator(validator_dict["comparator"]) comparator = utils.get_uniform_comparator(validator_dict["comparator"])
validate_func = self.testcase_parser.get_bind_function(comparator) validate_func = self.testcase_parser.get_bind_function(comparator)
httprunner/report.py
+6 -6
View File
@@ -38,8 +38,7 @@ def get_summary(result):
'skipped': len(result.skipped), 'skipped': len(result.skipped),
'expectedFailures': len(result.expectedFailures), 'expectedFailures': len(result.expectedFailures),
'unexpectedSuccesses': len(result.unexpectedSuccesses) 'unexpectedSuccesses': len(result.unexpectedSuccesses)
}, }
"platform": get_platform()
} }
summary["stat"]["successes"] = summary["stat"]["testsRun"] \ summary["stat"]["successes"] = summary["stat"]["testsRun"] \
- summary["stat"]["failures"] \ - summary["stat"]["failures"] \
@@ -90,10 +89,11 @@ def render_html_report(summary, html_report_name=None, html_report_template=None
if not os.path.isdir(report_dir_path): if not os.path.isdir(report_dir_path):
os.makedirs(report_dir_path) os.makedirs(report_dir_path)
for record in summary.get("records"): for suite_summary in summary["details"]:
meta_data = record['meta_data'] for record in suite_summary.get("records"):
stringify_body(meta_data, 'request') meta_data = record['meta_data']
stringify_body(meta_data, 'response') stringify_body(meta_data, 'request')
stringify_body(meta_data, 'response')
with io.open(html_report_template, "r", encoding='utf-8') as fp_r: with io.open(html_report_template, "r", encoding='utf-8') as fp_r:
template_content = fp_r.read() template_content = fp_r.read()
httprunner/task.py
+76 -64
View File
@@ -6,7 +6,8 @@ import unittest
from httprunner import exception, logger, runner, testcase, utils from httprunner import exception, logger, runner, testcase, utils
from httprunner.compat import is_py3 from httprunner.compat import is_py3
from httprunner.report import HtmlTestResult, get_summary, render_html_report from httprunner.report import (HtmlTestResult, get_platform, get_summary,
render_html_report)
from httprunner.testcase import TestcaseLoader from httprunner.testcase import TestcaseLoader
from httprunner.utils import load_dot_env_file from httprunner.utils import load_dot_env_file
@@ -29,6 +30,7 @@ class TestCase(unittest.TestCase):
self.meta_data = self.test_runner.http_client_session.meta_data self.meta_data = self.test_runner.http_client_session.meta_data
self.test_runner.http_client_session.init_meta_data() self.test_runner.http_client_session.init_meta_data()
class TestSuite(unittest.TestSuite): class TestSuite(unittest.TestSuite):
""" create test suite with a testset, it may include one or several testcases. """ create test suite with a testset, it may include one or several testcases.
each suite should initialize a separate Runner() with testset config. each suite should initialize a separate Runner() with testset config.
@@ -64,12 +66,12 @@ class TestSuite(unittest.TestSuite):
super(TestSuite, self).__init__() super(TestSuite, self).__init__()
self.test_runner_list = [] self.test_runner_list = []
config_dict = testset.get("config", {}) self.config = testset.get("config", {})
self.output_variables_list = config_dict.get("output", []) self.output_variables_list = self.config.get("output", [])
self.testset_file_path = config_dict.get("path") self.testset_file_path = self.config.get("path")
config_dict_parameters = config_dict.get("parameters", []) config_dict_parameters = self.config.get("parameters", [])
config_dict_variables = config_dict.get("variables", []) config_dict_variables = self.config.get("variables", [])
variables_mapping = variables_mapping or {} variables_mapping = variables_mapping or {}
config_dict_variables = utils.override_variables_binds(config_dict_variables, variables_mapping) config_dict_variables = utils.override_variables_binds(config_dict_variables, variables_mapping)
@@ -82,8 +84,8 @@ class TestSuite(unittest.TestSuite):
for config_variables in config_parametered_variables_list: for config_variables in config_parametered_variables_list:
# config level # config level
config_dict["variables"] = config_variables self.config["variables"] = config_variables
test_runner = runner.Runner(config_dict, http_client_session) test_runner = runner.Runner(self.config, http_client_session)
for testcase_dict in testcases: for testcase_dict in testcases:
testcase_dict = copy.copy(testcase_dict) testcase_dict = copy.copy(testcase_dict)
@@ -148,55 +150,30 @@ class TestSuite(unittest.TestSuite):
if not out: if not out:
continue continue
outputs.append({"in": variables, "out": out}) in_out = {"in": variables, "out": out}
if in_out not in outputs:
outputs.append(in_out)
return outputs return outputs
class TaskSuite(unittest.TestSuite):
""" create task suite with specified testcase path.
each task suite may include one or several test suite.
"""
def __init__(self, testsets, mapping=None, http_client_session=None):
"""
@params
testsets (dict/list): testset or list of testset
testset_dict
or
[
testset_dict_1,
testset_dict_2,
{
"name": "desc1",
"config": {},
"api": {},
"testcases": [testcase11, testcase12]
}
]
mapping (dict):
passed in variables mapping, it will override variables in config block
"""
super(TaskSuite, self).__init__()
mapping = mapping or {}
if not testsets: def init_test_suites(path_or_testsets, mapping=None, http_client_session=None):
raise exception.TestcaseNotFound """ initialize TestSuite list with testset path or testset dict
@params
if isinstance(testsets, dict): testsets (dict/list): testset or list of testset
testsets = [testsets] testset_dict
or
self.suite_list = [] [
for testset in testsets: testset_dict_1,
suite = TestSuite(testset, mapping, http_client_session) testset_dict_2,
self.addTest(suite) {
self.suite_list.append(suite) "config": {},
"api": {},
@property "testcases": [testcase11, testcase12]
def tasks(self): }
return self.suite_list ]
mapping (dict):
passed in variables mapping, it will override variables in config block
def init_task_suite(path_or_testsets, mapping=None, http_client_session=None):
""" initialize task suite
""" """
if not testcase.is_testsets(path_or_testsets): if not testcase.is_testsets(path_or_testsets):
TestcaseLoader.load_test_dependencies() TestcaseLoader.load_test_dependencies()
@@ -206,7 +183,19 @@ def init_task_suite(path_or_testsets, mapping=None, http_client_session=None):
# TODO: move comparator uniform here # TODO: move comparator uniform here
mapping = mapping or {} mapping = mapping or {}
return TaskSuite(testsets, mapping, http_client_session)
if not testsets:
raise exception.TestcaseNotFound
if isinstance(testsets, dict):
testsets = [testsets]
test_suite_list = []
for testset in testsets:
test_suite = TestSuite(testset, mapping, http_client_session)
test_suite_list.append(test_suite)
return test_suite_list
class HttpRunner(object): class HttpRunner(object):
@@ -242,19 +231,42 @@ class HttpRunner(object):
if mapping specified, it will override variables in config block if mapping specified, it will override variables in config block
""" """
try: try:
task_suite = init_task_suite(path_or_testsets, mapping) test_suite_list = init_test_suites(path_or_testsets, mapping)
except exception.TestcaseNotFound: except exception.TestcaseNotFound:
logger.log_error("Testcases not found in {}".format(path_or_testsets)) logger.log_error("Testcases not found in {}".format(path_or_testsets))
sys.exit(1) sys.exit(1)
result = self.runner.run(task_suite) self.summary = {
self.summary = get_summary(result) "success": True,
"stat": {},
"time": {},
"platform": get_platform(),
"details": []
}
output = [] def accumulate_stat(origin_stat, new_stat):
for task in task_suite.tasks: """ accumulate new_stat to origin_stat
output.extend(task.output) """
for key in new_stat:
if key not in origin_stat:
origin_stat[key] = new_stat[key]
elif key == "start_at":
# start datetime
origin_stat[key] = min(origin_stat[key], new_stat[key])
else:
origin_stat[key] += new_stat[key]
for test_suite in test_suite_list:
result = self.runner.run(test_suite)
test_suite_summary = get_summary(result)
self.summary["success"] &= test_suite_summary["success"]
test_suite_summary["name"] = test_suite.config.get("name")
test_suite_summary["base_url"] = test_suite.config.get("request", {}).get("base_url", "")
test_suite_summary["output"] = test_suite.output
accumulate_stat(self.summary["stat"], test_suite_summary["stat"])
accumulate_stat(self.summary["time"], test_suite_summary["time"])
self.summary["details"].append(test_suite_summary)
self.summary["output"] = output
return self return self
def gen_html_report(self, html_report_name=None, html_report_template=None): def gen_html_report(self, html_report_name=None, html_report_template=None):
@@ -274,11 +286,11 @@ class HttpRunner(object):
class LocustTask(object): class LocustTask(object):
def __init__(self, path_or_testsets, locust_client, mapping=None): def __init__(self, path_or_testsets, locust_client, mapping=None):
self.task_suite = init_task_suite(path_or_testsets, mapping, locust_client) self.test_suite_list = init_test_suites(path_or_testsets, mapping, locust_client)
def run(self): def run(self):
for suite in self.task_suite: for test_suite in self.test_suite_list:
for test in suite: for test in test_suite:
try: try:
test.runTest() test.runTest()
except exception.MyBaseError as ex: except exception.MyBaseError as ex:
httprunner/templates/default_report_template.html
+51 -29
View File
@@ -9,7 +9,7 @@
margin: 0 auto; margin: 0 auto;
width: 960px; width: 960px;
} }
#summary, #details { #summary {
width: 960px; width: 960px;
margin-bottom: 20px; margin-bottom: 20px;
} }
@@ -22,30 +22,34 @@
text-align: center; text-align: center;
padding: 4px 8px; padding: 4px 8px;
} }
#details th { .details {
width: 960px;
margin-bottom: 20px;
}
.details th {
background-color: skyblue; background-color: skyblue;
padding: 5px 12px; padding: 5px 12px;
} }
#details td { .details td {
background-color: lightblue; background-color: lightblue;
padding: 5px 12px; padding: 5px 12px;
} }
#details .detail { .details .detail {
background-color: lightgrey; background-color: lightgrey;
font-size: smaller; font-size: smaller;
padding: 5px 10px; padding: 5px 10px;
text-align: center; text-align: center;
} }
#details .success { .details .success {
background-color: greenyellow; background-color: greenyellow;
} }
#details .error { .details .error {
background-color: red; background-color: red;
} }
#details .failure { .details .failure {
background-color: salmon; background-color: salmon;
} }
#details .skipped { .details .skipped {
background-color: gray; background-color: gray;
} }
@@ -170,25 +174,42 @@
</table> </table>
<h2>Details</h2> <h2>Details</h2>
<table id="details">
<tr> {% for test_suite_summary in details %}
<th>Status</th> {% set suite_index = loop.index %}
<th>Name</th> <h3>{{test_suite_summary.name}}</h3>
<th>Response Time</th> <table class="details">
<th>Detail</th> <tr>
</tr> <th>base_url</th>
{% for record in records %} <td colspan="4">{{test_suite_summary.base_url}}</td>
<tr id="record_{{loop.index}}"> </tr>
<tr>
<td>TOTAL: {{test_suite_summary.stat.testsRun}}</td>
<td>SUCCESS: {{test_suite_summary.stat.successes}}</td>
<td>FAILED: {{test_suite_summary.stat.failures}}</td>
<td>ERROR: {{test_suite_summary.stat.errors}}</td>
<td>SKIPPED: {{test_suite_summary.stat.skipped}}</td>
</tr>
<tr>
<th>Status</th>
<th colspan="2">Name</th>
<th>Response Time</th>
<th>Detail</th>
</tr>
{% for record in test_suite_summary.records %}
{% set record_index = "{}_{}".format(suite_index, loop.index) %}
<tr id="record_{{record_index}}">
<th class="{{record.status}}" style="width:5em;">{{record.status}}</td> <th class="{{record.status}}" style="width:5em;">{{record.status}}</td>
<td>{{record.name}}</td> <td colspan="2">{{record.name}}</td>
<td style="text-align:center;width:6em;">{{ record.meta_data["response_time_ms"] }} ms</td> <td style="text-align:center;width:6em;">{{ record.meta_data.response_time_ms }} ms</td>
<td class="detail"> <td class="detail">
<a class="button" href="#popup_log_{{loop.index}}">log</a> <a class="button" href="#popup_log_{{record_index}}">log</a>
<div id="popup_log_{{loop.index}}" class="overlay"> <div id="popup_log_{{record_index}}" class="overlay">
<div class="popup"> <div class="popup">
<h2>Request and Response data</h2> <h2>Request and Response data</h2>
<a class="close" href="#record_{{loop.index}}">&times;</a> <a class="close" href="#record_{{record_index}}">&times;</a>
<div class="content"> <div class="content">
<h3>Request:</h3> <h3>Request:</h3>
@@ -261,15 +282,15 @@
<table> <table>
<tr> <tr>
<th>content_size(bytes)</th> <th>content_size(bytes)</th>
<td>{{ record.meta_data["content_size"] }}</td> <td>{{ record.meta_data.content_size }}</td>
</tr> </tr>
<tr> <tr>
<th>response_time(ms)</th> <th>response_time(ms)</th>
<td>{{ record.meta_data["response_time_ms"] }}</td> <td>{{ record.meta_data.response_time_ms }}</td>
</tr> </tr>
<tr> <tr>
<th>elapsed(ms)</th> <th>elapsed(ms)</th>
<td>{{ record.meta_data["elapsed_ms"] }}</td> <td>{{ record.meta_data.elapsed_ms }}</td>
</tr> </tr>
</table> </table>
</div> </div>
@@ -279,11 +300,11 @@
</div> </div>
{% if record.attachment %} {% if record.attachment %}
<a class="button" href="#popup_attachment_{{loop.index}}">traceback</a> <a class="button" href="#popup_attachment_{{record_index}}">traceback</a>
<div id="popup_attachment_{{loop.index}}" class="overlay"> <div id="popup_attachment_{{record_index}}" class="overlay">
<div class="popup"> <div class="popup">
<h2>Traceback Message</h2> <h2>Traceback Message</h2>
<a class="close" href="#record_{{loop.index}}">&times;</a> <a class="close" href="#record_{{record_index}}">&times;</a>
<div class="content"><pre>{{ record.attachment }}</pre></div> <div class="content"><pre>{{ record.attachment }}</pre></div>
</div> </div>
</div> </div>
@@ -292,5 +313,6 @@
</td> </td>
</tr> </tr>
{% endfor %} {% endfor %}
</table> </table>
{% endfor %}
</body> </body>
httprunner/testcase.py
-3
View File
@@ -206,13 +206,11 @@ class TestcaseLoader(object):
] ]
@return testset dict @return testset dict
{ {
"name": "desc1",
"config": {}, "config": {},
"testcases": [testcase11, testcase12] "testcases": [testcase11, testcase12]
} }
""" """
testset = { testset = {
"name": "",
"config": { "config": {
"path": file_path "path": file_path
}, },
@@ -228,7 +226,6 @@ class TestcaseLoader(object):
if key == "config": if key == "config":
testset["config"].update(test_block) testset["config"].update(test_block)
testset["name"] = test_block.get("name", "")
elif key == "test": elif key == "test":
if "api" in test_block: if "api" in test_block:
tests/test_httprunner.py
+6 -4
View File
@@ -90,7 +90,8 @@ class TestHttpRunner(ApiServerUnittest):
summary = runner.summary summary = runner.summary
self.assertTrue(summary["success"]) self.assertTrue(summary["success"])
self.assertEqual(summary["stat"]["testsRun"], 2) self.assertEqual(summary["stat"]["testsRun"], 2)
self.assertIn("records", summary) self.assertIn("details", summary)
self.assertIn("records", summary["details"][0])
def test_run_testset(self): def test_run_testset(self):
testsets = self.testset testsets = self.testset
@@ -98,7 +99,7 @@ class TestHttpRunner(ApiServerUnittest):
summary = runner.summary summary = runner.summary
self.assertTrue(summary["success"]) self.assertTrue(summary["success"])
self.assertEqual(summary["stat"]["testsRun"], 2) self.assertEqual(summary["stat"]["testsRun"], 2)
self.assertIn("records", summary) self.assertIn("records", summary["details"][0])
def test_run_yaml_upload(self): def test_run_yaml_upload(self):
testset_path = "tests/httpbin/upload.yml" testset_path = "tests/httpbin/upload.yml"
@@ -106,7 +107,8 @@ class TestHttpRunner(ApiServerUnittest):
summary = runner.summary summary = runner.summary
self.assertTrue(summary["success"]) self.assertTrue(summary["success"])
self.assertEqual(summary["stat"]["testsRun"], 1) self.assertEqual(summary["stat"]["testsRun"], 1)
self.assertIn("records", summary) self.assertIn("details", summary)
self.assertIn("records", summary["details"][0])
def test_run_post_data(self): def test_run_post_data(self):
testsets = [ testsets = [
@@ -135,7 +137,7 @@ class TestHttpRunner(ApiServerUnittest):
summary = runner.summary summary = runner.summary
self.assertTrue(summary["success"]) self.assertTrue(summary["success"])
self.assertEqual(summary["stat"]["testsRun"], 1) self.assertEqual(summary["stat"]["testsRun"], 1)
self.assertEqual(summary["records"][0]["meta_data"]["response_body"]["data"], "abc") self.assertEqual(summary["details"][0]["records"][0]["meta_data"]["response_body"]["data"], "abc")
def test_html_report_repsonse_image(self): def test_html_report_repsonse_image(self):
testset_path = "tests/httpbin/load_image.yml" testset_path = "tests/httpbin/load_image.yml"
tests/test_runner.py
+7 -5
View File
@@ -282,8 +282,9 @@ class TestRunner(ApiServerUnittest):
runner = HttpRunner().run(testcase_file_path) runner = HttpRunner().run(testcase_file_path)
summary = runner.summary summary = runner.summary
self.assertTrue(summary["success"]) self.assertTrue(summary["success"])
self.assertIn("token", summary["output"][0]["out"]) self.assertIn("token", summary["details"][0]["output"][0]["out"])
self.assertEqual(len(summary["output"]), 13) #TODO: fix
self.assertEqual(len(summary["details"][0]["output"]), 3)
def test_run_testset_with_variables_mapping(self): def test_run_testset_with_variables_mapping(self):
testcase_file_path = os.path.join( testcase_file_path = os.path.join(
@@ -294,8 +295,9 @@ class TestRunner(ApiServerUnittest):
runner = HttpRunner().run(testcase_file_path, mapping=variables_mapping) runner = HttpRunner().run(testcase_file_path, mapping=variables_mapping)
summary = runner.summary summary = runner.summary
self.assertTrue(summary["success"]) self.assertTrue(summary["success"])
self.assertIn("token", summary["output"][0]["out"]) self.assertIn("token", summary["details"][0]["output"][0]["out"])
self.assertEqual(len(summary["output"]), 13) #TODO: fix
self.assertEqual(len(summary["details"][0]["output"]), 3)
def test_run_testcase_with_empty_header(self): def test_run_testcase_with_empty_header(self):
testcase_file_path = os.path.join( testcase_file_path = os.path.join(
@@ -328,5 +330,5 @@ class TestRunner(ApiServerUnittest):
runner = HttpRunner().run(testcase_file_path) runner = HttpRunner().run(testcase_file_path)
summary = runner.summary summary = runner.summary
self.assertTrue(summary["success"]) self.assertTrue(summary["success"])
self.assertEqual(len(summary["output"]), 3 * 2 * 2) self.assertEqual(len(summary["details"][0]["output"]), 3 * 2 * 2)
self.assertEqual(summary["stat"]["testsRun"], 3 * 2 * 2) self.assertEqual(summary["stat"]["testsRun"], 3 * 2 * 2)
tests/test_task.py
+5 -4
View File
@@ -73,8 +73,9 @@ class TestTask(ApiServerUnittest):
] ]
} }
] ]
task_suite = task.TaskSuite(testsets) test_suite_list = task.init_test_suites(testsets)
self.assertEqual(len(test_suite_list), 1)
task_suite = test_suite_list[0]
self.assertEqual(task_suite.countTestCases(), 2) self.assertEqual(task_suite.countTestCases(), 2)
for suite in task_suite: for testcase in task_suite:
for testcase in suite: self.assertIsInstance(testcase, task.TestCase)
self.assertIsInstance(testcase, task.TestCase)
tests/test_testcase.py
+2 -3
View File
@@ -34,7 +34,6 @@ class TestTestcaseLoader(unittest.TestCase):
def test_load_test_file_suite(self): def test_load_test_file_suite(self):
TestcaseLoader.load_api_file("tests/api/basic.yml") TestcaseLoader.load_api_file("tests/api/basic.yml")
testset = TestcaseLoader.load_test_file("tests/suite/create_and_get.yml") testset = TestcaseLoader.load_test_file("tests/suite/create_and_get.yml")
self.assertEqual(testset["name"], "create user and check result.")
self.assertEqual(testset["config"]["name"], "create user and check result.") self.assertEqual(testset["config"]["name"], "create user and check result.")
self.assertEqual(len(testset["testcases"]), 3) self.assertEqual(len(testset["testcases"]), 3)
self.assertEqual(testset["testcases"][0]["name"], "make sure user $uid does not exist") self.assertEqual(testset["testcases"][0]["name"], "make sure user $uid does not exist")
@@ -43,7 +42,7 @@ class TestTestcaseLoader(unittest.TestCase):
def test_load_test_file_testcase(self): def test_load_test_file_testcase(self):
TestcaseLoader.load_test_dependencies() TestcaseLoader.load_test_dependencies()
testset = TestcaseLoader.load_test_file("tests/testcases/smoketest.yml") testset = TestcaseLoader.load_test_file("tests/testcases/smoketest.yml")
self.assertEqual(testset["name"], "smoketest") self.assertEqual(testset["config"]["name"], "smoketest")
self.assertEqual(testset["config"]["path"], "tests/testcases/smoketest.yml") self.assertEqual(testset["config"]["path"], "tests/testcases/smoketest.yml")
self.assertIn("device_sn", testset["config"]["variables"][0]) self.assertIn("device_sn", testset["config"]["variables"][0])
self.assertEqual(len(testset["testcases"]), 8) self.assertEqual(len(testset["testcases"]), 8)
@@ -76,7 +75,7 @@ class TestTestcaseLoader(unittest.TestCase):
def test_get_test_definition_suite(self): def test_get_test_definition_suite(self):
TestcaseLoader.load_test_dependencies() TestcaseLoader.load_test_dependencies()
api_def = TestcaseLoader._get_test_definition("create_and_check", "suite") api_def = TestcaseLoader._get_test_definition("create_and_check", "suite")
self.assertEqual(api_def["name"], "create user and check result.") self.assertEqual(api_def["config"]["name"], "create user and check result.")
with self.assertRaises(SuiteNotFound): with self.assertRaises(SuiteNotFound):
TestcaseLoader._get_test_definition("create_and_check_XXX", "suite") TestcaseLoader._get_test_definition("create_and_check_XXX", "suite")