diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 00c0029c..e322f1c9 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'HTTP test runner, not just about api test and load test.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.0.0' +__version__ = '1.1.0-beta' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/task.py b/httprunner/task.py index f2ffbc53..18a2dc3d 100644 --- a/httprunner/task.py +++ b/httprunner/task.py @@ -1,3 +1,4 @@ +import copy import sys import unittest @@ -32,12 +33,15 @@ class TestSuite(unittest.TestSuite): "name": "testset description", "requires": [], "function_binds": {}, + "parameters": {}, "variables": [], - "request": {} + "request": {}, + "output": [] }, "testcases": [ { "name": "testcase description", + "parameters": {}, "variables": [], # optional, override "request": {}, "extract": {}, # optional @@ -51,44 +55,92 @@ class TestSuite(unittest.TestSuite): """ def __init__(self, testset, variables_mapping=None, http_client_session=None): super(TestSuite, self).__init__() + self.test_runner_list = [] - self.config_dict = testset.get("config", {}) + config_dict = testset.get("config", {}) + self.output_variables_list = config_dict.get("output", []) + self.testset_file_path = config_dict["path"] + config_dict_parameters = config_dict.get("parameters", []) - variables = self.config_dict.get("variables", []) + config_dict_variables = config_dict.get("variables", []) variables_mapping = variables_mapping or {} - self.config_dict["variables"] = utils.override_variables_binds(variables, variables_mapping) + config_dict_variables = utils.override_variables_binds(config_dict_variables, variables_mapping) - parameters = self.config_dict.get("parameters", []) + config_parametered_variables_list = self._get_parametered_variables( + config_dict_variables, + config_dict_parameters + ) + self.testcase_parser = testcase.TestcaseParser() + testcases = testset.get("testcases", []) + + for config_variables in config_parametered_variables_list: + # config level + config_dict["variables"] = config_variables + test_runner = runner.Runner(config_dict, http_client_session) + + for testcase_dict in testcases: + testcase_dict = copy.copy(testcase_dict) + # testcase level + testcase_parametered_variables_list = self._get_parametered_variables( + testcase_dict.get("variables", []), + testcase_dict.get("parameters", []) + ) + for testcase_variables in testcase_parametered_variables_list: + testcase_dict["variables"] = testcase_variables + + # eval testcase name with bind variables + variables = utils.override_variables_binds( + config_variables, + testcase_variables + ) + self.testcase_parser.update_binded_variables(variables) + testcase_name = self.testcase_parser.eval_content_with_bindings(testcase_dict["name"]) + self.test_runner_list.append((test_runner, variables)) + + self._add_test_to_suite(testcase_name, test_runner, testcase_dict) + + def _get_parametered_variables(self, variables, parameters): + """ parameterize varaibles with parameters + """ cartesian_product_parameters = testcase.parse_parameters( parameters, - self.config_dict["path"] + self.testset_file_path ) or [{}] + + parametered_variables_list = [] for parameter_mapping in cartesian_product_parameters: - if parameter_mapping: - self.config_dict["variables"] = utils.override_variables_binds( - self.config_dict["variables"], - parameter_mapping - ) + parameter_mapping = parameter_mapping or {} + variables = utils.override_variables_binds( + variables, + parameter_mapping + ) - self.test_runner = runner.Runner(self.config_dict, http_client_session) - testcases = testset.get("testcases", []) - self._add_tests_to_suite(testcases) + parametered_variables_list.append(variables) - def _add_tests_to_suite(self, testcases): - for testcase_dict in testcases: - testcase_name = self.test_runner.context.eval_content(testcase_dict["name"]) - if utils.PYTHON_VERSION == 3: - TestCase.runTest.__doc__ = testcase_name - else: - TestCase.runTest.__func__.__doc__ = testcase_name + return parametered_variables_list - test = TestCase(self.test_runner, testcase_dict) - [self.addTest(test) for _ in range(int(testcase_dict.get("times", 1)))] + def _add_test_to_suite(self, testcase_name, test_runner, testcase_dict): + if utils.PYTHON_VERSION == 3: + TestCase.runTest.__doc__ = testcase_name + else: + TestCase.runTest.__func__.__doc__ = testcase_name + + test = TestCase(test_runner, testcase_dict) + [self.addTest(test) for _ in range(int(testcase_dict.get("times", 1)))] @property def output(self): - output_variables_list = self.config_dict.get("output", []) - return self.test_runner.extract_output(output_variables_list) + outputs = [] + + for test_runner, variables in self.test_runner_list: + outputs.append( + { + "in": variables, + "out": test_runner.extract_output(self.output_variables_list) + } + ) + + return outputs class TaskSuite(unittest.TestSuite): """ create task suite with specified testcase path. @@ -167,9 +219,9 @@ class HttpRunner(object): result = self.runner.run(task_suite) - output = {} + output = [] for task in task_suite.tasks: - output.update(task.output) + output.extend(task.output) if self.gen_html_report: summary = result.summary diff --git a/httprunner/utils.py b/httprunner/utils.py index 9afabc6b..b47e8d46 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -1,3 +1,4 @@ +import copy import hashlib import hmac import imp @@ -319,10 +320,11 @@ def update_ordered_dict(ordered_dict, override_mapping): "c": 4 }) """ + new_ordered_dict = copy.copy(ordered_dict) for var, value in override_mapping.items(): - ordered_dict.update({var: value}) + new_ordered_dict.update({var: value}) - return ordered_dict + return new_ordered_dict def override_variables_binds(variables, new_mapping): """ convert variables in testcase to ordered mapping, with new_mapping overrided @@ -339,25 +341,40 @@ def override_variables_binds(variables, new_mapping): new_mapping ) -def print_output(output): - if not output: +def print_output(outputs): + + if not outputs: return - content = "\n================== Output ==================\n" - content += '{:<16}: {:<}\n'.format("Variable", "Value") - content += '{:<16}: {:<}\n'.format("--------", "-----") + content = "\n================== Variables & Output ==================\n" + content += '{:<6} | {:<16} : {:<}\n'.format("Type", "Variable", "Value") + content += '{:<6} | {:<16} : {:<}\n'.format("-" * 6, "-" * 16, "-" * 27) - for variable, value in output.items(): + def prepare_content(var_type, in_out): + content = "" + for variable, value in in_out.items(): - if PYTHON_VERSION == 2: - if isinstance(variable, unicode): - variable = variable.encode("utf-8") - if isinstance(value, unicode): - value = value.encode("utf-8") + if PYTHON_VERSION == 2: + if isinstance(variable, unicode): + variable = variable.encode("utf-8") + if isinstance(value, unicode): + value = value.encode("utf-8") - content += '{:<16}: {:<}\n'.format(variable, value) + content += '{:<6} | {:<16} : {:<}\n'.format(var_type, variable, value) - content += "============================================\n" + return content + + for output in outputs: + _in = output["in"] + _out = output["out"] + + if not _out: + continue + + content += prepare_content("Var", _in) + content += "\n" + content += prepare_content("Out", _out) + content += "-" * 56 + "\n" logger.log_debug(content) diff --git a/tests/data/demo_parameters.yml b/tests/data/demo_parameters.yml index bc08e408..8dac2e08 100644 --- a/tests/data/demo_parameters.yml +++ b/tests/data/demo_parameters.yml @@ -2,10 +2,7 @@ name: "user management testset." parameters: - user_agent: ["iOS/10.1", "iOS/10.2", "iOS/10.3"] - - app_version: ${gen_app_version()} - - username: ${parameterize(account.csv)} variables: - - user_agent: 'iOS/10.3' - device_sn: ${gen_random_string(15)} - os_platform: 'ios' request: @@ -17,10 +14,22 @@ - token - test: - name: get token with $user_agent and $app_version, username $username + name: get token with $user_agent and $app_version + parameters: + - app_version: ${gen_app_version()} api: get_token($user_agent, $device_sn, $os_platform, $app_version) extract: - token: content.token validate: - "eq": ["status_code", 200] - "len_eq": ["content.token", 16] + +# - test: +# name: create user +# parameters: +# - user_id: [1001, 1002, 1003] +# - username-password: ${parameterize(account.csv)} +# api: create_user($user_id, $username, $password, $token) +# validate: +# - {"check": "status_code", "expect": 201} +# - {"check": "content.success", "expect": true} diff --git a/tests/data/demo_testset_layer.yml b/tests/data/demo_testset_layer.yml index bb2b9cd6..61723722 100644 --- a/tests/data/demo_testset_layer.yml +++ b/tests/data/demo_testset_layer.yml @@ -14,7 +14,7 @@ - token - test: - name: get token + name: get token with $user_agent, $app_version api: get_token($user_agent, $device_sn, $os_platform, $app_version) extract: - token: content.token diff --git a/tests/test_runner.py b/tests/test_runner.py index 4079aac6..16d0dff8 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -121,7 +121,8 @@ class TestRunner(ApiServerUnittest): os.getcwd(), 'tests/data/demo_testset_layer.yml') result = HttpRunner(testcase_file_path).run() self.assertTrue(result["success"]) - self.assertIn("token", result["output"]) + self.assertIn("token", result["output"][0]["out"]) + self.assertEqual(len(result["output"]), 13) def test_run_testset_with_variables_mapping(self): testcase_file_path = os.path.join( @@ -131,7 +132,8 @@ class TestRunner(ApiServerUnittest): } result = HttpRunner(testcase_file_path).run(mapping=variables_mapping) self.assertTrue(result["success"]) - self.assertIn("token", result["output"]) + self.assertIn("token", result["output"][0]["out"]) + self.assertEqual(len(result["output"]), 13) def test_run_testcase_with_empty_header(self): testcase_file_path = os.path.join( @@ -163,5 +165,5 @@ class TestRunner(ApiServerUnittest): os.getcwd(), 'tests/data/demo_parameters.yml') result = HttpRunner(testcase_file_path).run() self.assertTrue(result["success"]) - self.assertIn("token", result["output"]) - self.assertEqual(result["stat"]["testsRun"], 3 * 2 * 3) + self.assertEqual(len(result["output"]), 3 * 2) + self.assertEqual(result["stat"]["testsRun"], 3 * 2)