From 9b25bd65fa95521b0d106ec63356fd417adda9ac Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 6 Nov 2018 14:44:54 +0800 Subject: [PATCH 001/113] remove unused loading tests --- httprunner/templates/locustfile_template | 1 - 1 file changed, 1 deletion(-) diff --git a/httprunner/templates/locustfile_template b/httprunner/templates/locustfile_template index 84381083..f5f481c5 100644 --- a/httprunner/templates/locustfile_template +++ b/httprunner/templates/locustfile_template @@ -16,7 +16,6 @@ logging.getLogger('locust.runners').setLevel(logging.INFO) class WebPageTasks(TaskSet): def on_start(self): self.test_runner = Runner(self.locust.config, self.client) - self.testcases = load_locust_tests(self.locust.file_path) @task(weight=1) def test_any(self): From 547e22a6a5edcfb7c5e05e54cd2b8f0c89884245 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 6 Nov 2018 23:44:46 +0800 Subject: [PATCH 002/113] fix merge & override api variables/validate/extract/hooks/etc --- httprunner/__about__.py | 2 +- httprunner/loader.py | 167 ++++++++++++++++++++++++---------------- tests/test_loader.py | 53 +++++++------ 3 files changed, 133 insertions(+), 89 deletions(-) diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 211c90c2..50405f44 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.5.15' +__version__ = '1.6.0.alpha' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/loader.py b/httprunner/loader.py index d8294c40..82aa5b4b 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -1,4 +1,5 @@ import collections +import copy import csv import importlib import io @@ -10,7 +11,6 @@ import yaml from httprunner import built_in, exceptions, logger, parser, utils, validator from httprunner.compat import OrderedDict - ############################################################################### ## file loader ############################################################################### @@ -235,6 +235,7 @@ def load_python_module(module): } """ + # TODO (2.0): remove variables from debugtalk.py debugtalk_module = { "variables": {}, "functions": {} @@ -324,46 +325,46 @@ def _load_teststeps(test_block, project_mapping): { "name": "add product to cart", "api": "api_add_cart()", - "validate": [] + "variables": [], + "validate": [], + "extract": [] } # testcase reference { "name": "add product to cart", "suite": "create_and_check()", - "validate": [] + "variables": [] } # define directly { "name": "checkout cart", "request": {}, - "validate": [] + "variables": [], + "validate": [], + "extract": [] } Returns: list: loaded teststeps list """ - def extend_api_definition(block): - ref_call = block["api"] - def_block = _get_block_by_name(ref_call, "def-api", project_mapping) - _extend_block(block, def_block) - teststeps = [] # reference api if "api" in test_block: - extend_api_definition(test_block) - teststeps.append(test_block) + ref_call = test_block.pop("api") + def_block = _get_block_by_name(ref_call, "def-api", project_mapping) + extended_block = _extend_block(test_block, def_block) + teststeps.append(extended_block) # reference testcase elif "suite" in test_block: # TODO: replace suite with testcase - ref_call = test_block["suite"] - block = _get_block_by_name(ref_call, "def-testcase", project_mapping) + ref_call = test_block.pop("suite") + def_block = _get_block_by_name(ref_call, "def-testcase", project_mapping) # TODO: bugfix lost block config variables - for teststep in block["teststeps"]: - if "api" in teststep: - extend_api_definition(teststep) - teststeps.append(teststep) + for teststep in def_block["teststeps"]: + _teststeps = _load_teststeps(teststep, project_mapping) + teststeps.extend(_teststeps) # define directly else: @@ -491,6 +492,8 @@ def _get_test_definition(name, ref_type, project_mapping): """ block = project_mapping.get(ref_type, {}).get(name) + # NOTICE: avoid project_mapping been changed during iteration. + block = copy.deepcopy(block) if not block: err_msg = "{} not found!".format(name) @@ -504,7 +507,7 @@ def _get_test_definition(name, ref_type, project_mapping): def _extend_block(ref_block, def_block): - """ extend ref_block with def_block. + """ extend ref_block with def_block, ref_block will merge and override def_block. Args: def_block (dict): api definition dict. @@ -533,27 +536,59 @@ def _extend_block(ref_block, def_block): } """ - # TODO: override variables - def_validators = def_block.get("validate") or def_block.get("validators", []) - ref_validators = ref_block.get("validate") or ref_block.get("validators", []) + extended_block = {} - def_extrators = def_block.get("extract") \ - or def_block.get("extractors") \ - or def_block.get("extract_binds", []) - ref_extractors = ref_block.get("extract") \ - or ref_block.get("extractors") \ - or ref_block.get("extract_binds", []) + # override name + extended_block["name"] = ref_block.pop("name", None) or def_block.pop("name", "") - ref_block.update(def_block) - ref_block["validate"] = _merge_validator( + # override variables + def_variables = def_block.pop("variables", []) + ref_variables = ref_block.pop("variables", []) + extended_block["variables"] = _extend_variables( + def_variables, + ref_variables + ) + + # merge & override validators + def_validators = def_block.pop("validate", None) or def_block.pop("validators", []) + ref_validators = ref_block.pop("validate", None) or ref_block.pop("validators", []) + extended_block["validate"] = _extend_validators( def_validators, ref_validators ) - ref_block["extract"] = _merge_extractor( + + # merge & override extractors + def_extrators = def_block.pop("extract", None) \ + or def_block.pop("extractors", None) \ + or def_block.pop("extract_binds", []) + ref_extractors = ref_block.pop("extract", None) \ + or ref_block.pop("extractors", None) \ + or ref_block.pop("extract_binds", []) + extended_block["extract"] = _extend_variables( def_extrators, ref_extractors ) + # TODO: merge & override request + def_request = def_block.pop("request", {}) + ref_request = ref_block.pop("request", {}) + extended_block["request"] = def_request + + # merge & override setup_hooks + def_setup_hooks = def_block.pop("setup_hooks", []) + ref_setup_hooks = ref_block.pop("setup_hooks", []) + extended_block["setup_hooks"] = list(set(def_setup_hooks + ref_setup_hooks)) + # merge & override teardown_hooks + def_teardown_hooks = def_block.pop("teardown_hooks", []) + ref_teardown_hooks = ref_block.pop("teardown_hooks", []) + extended_block["teardown_hooks"] = list(set(def_teardown_hooks + ref_teardown_hooks)) + + # TODO: extend with other ref block items, e.g. times + # extended_block.update(def_block) + extended_block.update(ref_block) + + return extended_block + def _convert_validators_to_mapping(validators): """ convert validators list to mapping. @@ -592,20 +627,21 @@ def _convert_validators_to_mapping(validators): return validators_mapping -def _merge_validator(def_validators, ref_validators): - """ merge def_validators with ref_validators. +def _extend_validators(def_validators, ref_validators): + """ extend ref_validators with def_validators. + ref_validators will merge and override def_validators. Args: def_validators (list): ref_validators (list): Returns: - list: merged validators + list: extended validators Examples: >>> def_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] >>> ref_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] - >>> _merge_validator(def_validators, ref_validators) + >>> _extend_validators(def_validators, ref_validators) [ {"check": "v1", "expect": 201, "comparator": "eq"}, {"check": "s2", "expect": 16, "comparator": "len_eq"}, @@ -627,20 +663,21 @@ def _merge_validator(def_validators, ref_validators): return list(def_validators_mapping.values()) -def _merge_extractor(def_extrators, ref_extractors): - """ merge def_extrators with ref_extractors +def _extend_variables(def_variables, ref_variables): + """ extend ref_variables with def_variables. + ref_variables will merge and override def_variables. Args: - def_extrators (list): [{"var1": "val1"}, {"var2": "val2"}] - ref_extractors (list): [{"var1": "val111"}, {"var3": "val3"}] + def_variables (list): + ref_variables (list): Returns: - list: merged extractors + list: extended variables Examples: - >>> def_extrators = [{"var1": "val1"}, {"var2": "val2"}] - >>> ref_extractors = [{"var1": "val111"}, {"var3": "val3"}] - >>> _merge_extractor(def_extrators, ref_extractors) + >>> def_variables = [{"var1": "val1"}, {"var2": "val2"}] + >>> ref_variables = [{"var1": "val111"}, {"var3": "val3"}] + >>> _extend_variables(def_variables, ref_variables) [ {"var1": "val111"}, {"var2": "val2"}, @@ -648,35 +685,29 @@ def _merge_extractor(def_extrators, ref_extractors): ] """ - if not def_extrators: - return ref_extractors + if not def_variables: + return ref_variables - elif not ref_extractors: - return def_extrators + elif not ref_variables: + return def_variables else: - extractor_dict = OrderedDict() - for api_extrator in def_extrators: - if len(api_extrator) != 1: - logger.log_warning("incorrect extractor: {}".format(api_extrator)) + extended_variables_dict = OrderedDict() + for def_variable in def_variables: + var_name = list(def_variable.keys())[0] + extended_variables_dict[var_name] = def_variable[var_name] + + for ref_variable in ref_variables: + if not ref_variable: continue + var_name = list(ref_variable.keys())[0] + extended_variables_dict[var_name] = ref_variable[var_name] - var_name = list(api_extrator.keys())[0] - extractor_dict[var_name] = api_extrator[var_name] + extended_variables = [] + for key, value in extended_variables_dict.items(): + extended_variables.append({key: value}) - for test_extrator in ref_extractors: - if len(test_extrator) != 1: - logger.log_warning("incorrect extractor: {}".format(test_extrator)) - continue - - var_name = list(test_extrator.keys())[0] - extractor_dict[var_name] = test_extrator[var_name] - - extractor_list = [] - for key, value in extractor_dict.items(): - extractor_list.append({key: value}) - - return extractor_list + return extended_variables def load_folder_content(folder_path): @@ -743,6 +774,7 @@ def load_api_folder(api_folder_path): } """ + # TODO: refactor api storage format, use one file for each api. api_definition_mapping = {} api_items_mapping = load_folder_content(api_folder_path) @@ -752,6 +784,7 @@ def load_api_folder(api_folder_path): for api_item in api_items: key, api_dict = api_item.popitem() + # TODO: replace def with api file path api_def = api_dict.pop("def") function_meta = parser.parse_function(api_def) func_name = function_meta["func_name"] @@ -826,6 +859,7 @@ def load_test_folder(test_folder_path): test_definition_mapping[test_file_path] = testcase continue + # TODO: replace def with testcase file path testcase_def = block.pop("def") function_meta = parser.parse_function(testcase_def) func_name = function_meta["func_name"] @@ -837,6 +871,7 @@ def load_test_folder(test_folder_path): test_definition_mapping[func_name] = testcase else: # key == "test": + ### TODO: extend suite with api testcase["teststeps"].append(block) return test_definition_mapping @@ -1016,6 +1051,8 @@ def load_locust_tests(path, dot_env_path=None): project_mapping = load_project_tests(path, dot_env_path) config = { + "variables": project_mapping["debugtalk"]["variables"], + "functions": project_mapping["debugtalk"]["functions"], "refs": project_mapping } tests = [] diff --git a/tests/test_loader.py b/tests/test_loader.py index 5d0a2f08..77a23a1a 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -285,7 +285,7 @@ class TestSuiteLoader(unittest.TestCase): def test_load_teststeps(self): test_block = { - "name": "setup and reset all.", + "name": "setup and reset all (override).", "suite": "setup_and_reset($device_sn)", "output": ["token", "device_sn"] } @@ -322,21 +322,28 @@ class TestSuiteLoader(unittest.TestCase): ) test_block = { "name": "override block", + "times": 3, "variables": [ {"var": 123} ], 'request': { - 'url': '/api/get-token', 'method': 'POST', 'headers': {'user_agent': '$user_agent', 'device_sn': '$device_sn', 'os_platform': '$os_platform', 'app_version': '$app_version'}, 'json': {'sign': '${get_sign($user_agent, $device_sn, $os_platform, $app_version)}'}}, + 'url': '/api/get-token', + 'method': 'POST', + 'headers': {'user_agent': '$user_agent', 'device_sn': '$device_sn', 'os_platform': '$os_platform', 'app_version': '$app_version'}, + 'json': {'sign': '${get_sign($user_agent, $device_sn, $os_platform, $app_version)}'} + }, 'validate': [ {'eq': ['status_code', 201]}, {'len_eq': ['content.token', 32]} ] } - loader._extend_block(test_block, def_block) - self.assertEqual(test_block["name"], "override block") - self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, test_block["validate"]) - self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, test_block["validate"]) + extended_block = loader._extend_block(test_block, def_block) + self.assertEqual(extended_block["name"], "override block") + self.assertIn({'var': 123}, extended_block["variables"]) + self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, extended_block["validate"]) + self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, extended_block["validate"]) + self.assertEqual(extended_block["times"], 3) def test_get_test_definition_api(self): api_def = loader._get_test_definition("get_headers", "def-api", self.project_mapping) @@ -354,7 +361,7 @@ class TestSuiteLoader(unittest.TestCase): with self.assertRaises(exceptions.TestcaseNotFound): loader._get_test_definition("create_and_check_XXX", "def-testcase", self.project_mapping) - def test_merge_validator(self): + def test_extend_validators(self): def_validators = [ {'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"} @@ -364,21 +371,21 @@ class TestSuiteLoader(unittest.TestCase): {'len_eq': ['s3', 12]} ] - merged_validators = loader._merge_validator(def_validators, current_validators) + extended_validators = loader._extend_validators(def_validators, current_validators) self.assertIn( {"check": "v1", "expect": 201, "comparator": "eq"}, - merged_validators + extended_validators ) self.assertIn( {"check": "s2", "expect": 16, "comparator": "len_eq"}, - merged_validators + extended_validators ) self.assertIn( {"check": "s3", "expect": 12, "comparator": "len_eq"}, - merged_validators + extended_validators ) - def test_merge_validator_with_dict(self): + def test_extend_validators_with_dict(self): def_validators = [ {'eq': ["a", {"v": 1}]}, {'eq': [{"b": 1}, 200]} @@ -388,27 +395,27 @@ class TestSuiteLoader(unittest.TestCase): {'eq': [{"b": 1}, 201]} ] - merged_validators = loader._merge_validator(def_validators, current_validators) - self.assertEqual(len(merged_validators), 3) - self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'eq'}, merged_validators) - self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'eq'}, merged_validators) + extended_validators = loader._extend_validators(def_validators, current_validators) + self.assertEqual(len(extended_validators), 3) + self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'eq'}, extended_validators) + self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'eq'}, extended_validators) - def test_merge_extractor(self): - api_extrators = [{"var1": "val1"}, {"var2": "val2"}] - current_extractors = [{"var1": "val111"}, {"var3": "val3"}] + def test_extend_variables(self): + def_variables = [{"var1": "val1"}, {"var2": "val2"}] + ref_variables = [{"var1": "val111"}, {"var3": "val3"}] - merged_extractors = loader._merge_extractor(api_extrators, current_extractors) + extended_variables = loader._extend_variables(def_variables, ref_variables) self.assertIn( {"var1": "val111"}, - merged_extractors + extended_variables ) self.assertIn( {"var2": "val2"}, - merged_extractors + extended_variables ) self.assertIn( {"var3": "val3"}, - merged_extractors + extended_variables ) def test_load_testcases_by_path_files(self): From ff5862189c93b5c1ff0f5c007430184c64eeafa7 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 7 Nov 2018 01:23:16 +0800 Subject: [PATCH 003/113] bugfix: ignore output content in deque type --- httprunner/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/httprunner/utils.py b/httprunner/utils.py index fc09ebe1..8bf7700e 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -1,5 +1,6 @@ # encoding: utf-8 +import collections import copy import io import itertools @@ -367,7 +368,7 @@ def print_io(in_out): def prepare_content(var_type, in_out): content = "" for variable, value in in_out.items(): - if isinstance(value, tuple): + if isinstance(value, (tuple, collections.deque)): continue elif isinstance(value, (dict, list)): value = json.dumps(value) From ed964c83729657f711b4931681c962909022b783 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 7 Nov 2018 21:15:26 +0800 Subject: [PATCH 004/113] bugfix: catch AssertionError in load test --- httprunner/templates/locustfile_template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httprunner/templates/locustfile_template b/httprunner/templates/locustfile_template index f5f481c5..1b61b4be 100644 --- a/httprunner/templates/locustfile_template +++ b/httprunner/templates/locustfile_template @@ -23,7 +23,7 @@ class WebPageTasks(TaskSet): for teststep in teststeps: try: self.test_runner.run_test(teststep) - except (MyBaseError, MyBaseFailure) as ex: + except (AssertionError, MyBaseError, MyBaseFailure) as ex: request_failure.fire( request_type=teststep.get("request", {}).get("method"), name=teststep.get("name"), From 39ca9fcc8d932d1682b3e9d3f2b7e640bec2b4a7 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 7 Nov 2018 22:19:39 +0800 Subject: [PATCH 005/113] set locust default loglevel to WARNING --- httprunner/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httprunner/cli.py b/httprunner/cli.py index cfbdb0df..079eff17 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -129,7 +129,7 @@ def main_locust(): loglevel = sys.argv[loglevel_index] else: # default - loglevel = "INFO" + loglevel = "WARNING" logger.setup_logger(loglevel) From 731dac884c634f53feeeccb21c812fbf213a9e03 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 7 Nov 2018 22:20:17 +0800 Subject: [PATCH 006/113] load_locust_tests: parse config --- httprunner/loader.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index 82aa5b4b..700f7582 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -1052,8 +1052,7 @@ def load_locust_tests(path, dot_env_path=None): config = { "variables": project_mapping["debugtalk"]["variables"], - "functions": project_mapping["debugtalk"]["functions"], - "refs": project_mapping + "functions": project_mapping["debugtalk"]["functions"] } tests = [] for item in raw_testcase: @@ -1067,6 +1066,35 @@ def load_locust_tests(path, dot_env_path=None): for _ in range(weight): tests.append(teststeps) + # parse config variables + raw_config_variables = config.get("variables", []) + parsed_config_variables = parser.parse_data( + raw_config_variables, + project_mapping["debugtalk"]["variables"], + project_mapping["debugtalk"]["functions"] + ) + + # priority: passed in > debugtalk.py > parameters > variables + # override variables mapping with parameters mapping + config_variables = utils.override_mapping_list( + parsed_config_variables, {}) + # merge debugtalk.py module variables + config_variables.update(project_mapping["debugtalk"]["variables"]) + + # parse config name + config["name"] = parser.parse_data( + config.get("name", ""), + config_variables, + project_mapping["debugtalk"]["functions"] + ) + + # parse config request + config["request"] = parser.parse_data( + config.get("request", {}), + config_variables, + project_mapping["debugtalk"]["functions"] + ) + return { "config": config, "tests": tests From a3f28bae40864da9612eaadb782759ab2f772ffe Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 7 Nov 2018 22:27:35 +0800 Subject: [PATCH 007/113] add tests layer temporally, remove later. --- httprunner/loader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index 700f7582..24106311 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -933,9 +933,10 @@ def load_project_tests(test_path, dot_env_path=None): "functions": {} } - project_mapping["def-api"] = load_api_folder(os.path.join(project_working_directory, "api")) + # TODO: add tests layer temporally, remove later. + project_mapping["def-api"] = load_api_folder(os.path.join(project_working_directory, "tests", "api")) # TODO: replace suite with testcases - project_mapping["def-testcase"] = load_test_folder(os.path.join(project_working_directory, "suite")) + project_mapping["def-testcase"] = load_test_folder(os.path.join(project_working_directory, "tests", "suite")) return project_mapping From d8c39da787a15fa1e4f635715be23add10b21a77 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 8 Nov 2018 12:01:26 +0800 Subject: [PATCH 008/113] bugfix: display url when teststep failed in locust --- httprunner/templates/locustfile_template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httprunner/templates/locustfile_template b/httprunner/templates/locustfile_template index 1b61b4be..907086db 100644 --- a/httprunner/templates/locustfile_template +++ b/httprunner/templates/locustfile_template @@ -26,7 +26,7 @@ class WebPageTasks(TaskSet): except (AssertionError, MyBaseError, MyBaseFailure) as ex: request_failure.fire( request_type=teststep.get("request", {}).get("method"), - name=teststep.get("name"), + name=teststep.get("request", {}).get("url"), response_time=0, exception=ex ) From a9a82e52f3254e48df79732ac9597967c7bb5df9 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 14 Nov 2018 15:41:41 +0800 Subject: [PATCH 009/113] fix variables priority: passed in > .env > debugtalk.py > parameters > config variables --- httprunner/loader.py | 5 ++--- httprunner/parser.py | 35 ++++++++++++++++++++--------------- httprunner/utils.py | 40 ++++++++++++++++++++++++++++++++++------ tests/test_loader.py | 2 +- 4 files changed, 57 insertions(+), 25 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index 24106311..700f7582 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -933,10 +933,9 @@ def load_project_tests(test_path, dot_env_path=None): "functions": {} } - # TODO: add tests layer temporally, remove later. - project_mapping["def-api"] = load_api_folder(os.path.join(project_working_directory, "tests", "api")) + project_mapping["def-api"] = load_api_folder(os.path.join(project_working_directory, "api")) # TODO: replace suite with testcases - project_mapping["def-testcase"] = load_test_folder(os.path.join(project_working_directory, "tests", "suite")) + project_mapping["def-testcase"] = load_test_folder(os.path.join(project_working_directory, "suite")) return project_mapping diff --git a/httprunner/parser.py b/httprunner/parser.py index 6c22c7e3..481ddc9d 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -629,24 +629,29 @@ def parse_tests(testcases, variables_mapping=None): for parameter_mapping in cartesian_product_parameters_list: testcase_dict = utils.deepcopy_dict(testcase) config = testcase_dict.get("config") - - # parse config variables + # get config variables raw_config_variables = config.get("variables", []) - parsed_config_variables = parse_data( - raw_config_variables, - project_mapping["debugtalk"]["variables"], - project_mapping["debugtalk"]["functions"] - ) + raw_config_variables_mapping = utils.ensure_mapping_format(raw_config_variables) - # priority: passed in > debugtalk.py > parameters > variables - # override variables mapping with parameters mapping - config_variables = utils.override_mapping_list( - parsed_config_variables, parameter_mapping) - # merge debugtalk.py module variables + # priority: passed in > .env > debugtalk.py > parameters > variables + + config_variables = utils.deepcopy_dict(parameter_mapping) config_variables.update(project_mapping["debugtalk"]["variables"]) - # override variables mapping with passed in variables_mapping - config_variables = utils.override_mapping_list( - config_variables, variables_mapping) + config_variables.update(variables_mapping) + + for key, value in raw_config_variables_mapping.items(): + + if key in config_variables: + # passed in & .env & parameters + continue + else: + # config variables + parsed_value = parse_data( + value, + config_variables, + project_mapping["debugtalk"]["functions"] + ) + config_variables[key] = parsed_value testcase_dict["config"]["variables"] = config_variables diff --git a/httprunner/utils.py b/httprunner/utils.py index 8bf7700e..78e14c64 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -264,6 +264,39 @@ def update_ordered_dict(ordered_dict, override_mapping): return new_ordered_dict +def ensure_mapping_format(variables): + """ ensure variables are in mapping format. + + Args: + variables (list/dict/OrderedDict): original variables + + Returns: + OrderedDict: ensured variables in OrderedDict + + Examples: + >>> variables = [ + {"a": 1}, + {"b": 2} + ] + >>> print(ensure_mapping_format(variables)) + OrderDict( + { + "a": 1, + "b": 2 + } + ) + + """ + if isinstance(variables, list): + variables_ordered_dict = convert_mappinglist_to_orderdict(variables) + elif isinstance(variables, (OrderedDict, dict)): + variables_ordered_dict = variables + else: + raise exceptions.ParamsError("variables format error!") + + return variables_ordered_dict + + def override_mapping_list(variables, new_mapping): """ override variables with new mapping. @@ -298,12 +331,7 @@ def override_mapping_list(variables, new_mapping): ) """ - if isinstance(variables, list): - variables_ordered_dict = convert_mappinglist_to_orderdict(variables) - elif isinstance(variables, (OrderedDict, dict)): - variables_ordered_dict = variables - else: - raise exceptions.ParamsError("variables error!") + variables_ordered_dict = ensure_mapping_format(variables) return update_ordered_dict( variables_ordered_dict, diff --git a/tests/test_loader.py b/tests/test_loader.py index 77a23a1a..104a855c 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -565,7 +565,7 @@ class TestSuiteLoader(unittest.TestCase): path = os.path.join( os.getcwd(), 'tests/data/demo_locust.yml') locust_tests = loader.load_locust_tests(path) - self.assertEqual(locust_tests["config"]["refs"]["env"]["UserName"], "debugtalk") + self.assertEqual(locust_tests["config"]["variables"]["UserName"], "debugtalk") self.assertEqual(len(locust_tests["tests"]), 10) self.assertEqual(locust_tests["tests"][0][0]["name"], "index") self.assertEqual(locust_tests["tests"][9][0]["name"], "user-agent") From 1578b78f54d29fdbbe70e7ca4f61ef939ab59321 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 14 Nov 2018 16:15:04 +0800 Subject: [PATCH 010/113] ensure variables are in mapping format. --- httprunner/context.py | 12 +++--------- httprunner/response.py | 2 +- httprunner/utils.py | 30 +++--------------------------- tests/test_utils.py | 6 +++--- 4 files changed, 10 insertions(+), 40 deletions(-) diff --git a/httprunner/context.py b/httprunner/context.py index 7484a503..55bd1da6 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -15,12 +15,7 @@ class Context(object): """ # testcase level context ## TESTCASE_SHARED_VARIABLES_MAPPING and TESTCASE_SHARED_FUNCTIONS_MAPPING are unchangeable. - if isinstance(variables, list): - self.TESTCASE_SHARED_VARIABLES_MAPPING = utils.convert_mappinglist_to_orderdict(variables) - else: - # dict - self.TESTCASE_SHARED_VARIABLES_MAPPING = variables or OrderedDict() - + self.TESTCASE_SHARED_VARIABLES_MAPPING = utils.ensure_mapping_format(variables) self.TESTCASE_SHARED_FUNCTIONS_MAPPING = functions or OrderedDict() # testcase level request, will not change @@ -65,10 +60,9 @@ class Context(object): level (enum): "testcase" or "teststep" """ - if isinstance(variables, list): - variables = utils.convert_mappinglist_to_orderdict(variables) + variables_mapping = utils.ensure_mapping_format(variables) - for variable_name, variable_value in variables.items(): + for variable_name, variable_value in variables_mapping.items(): variable_eval_value = self.eval_content(variable_value) if level == "testcase": diff --git a/httprunner/response.py b/httprunner/response.py index 821181ba..07f3c868 100644 --- a/httprunner/response.py +++ b/httprunner/response.py @@ -221,7 +221,7 @@ class ResponseObject(object): logger.log_info("start to extract from response object.") extracted_variables_mapping = OrderedDict() - extract_binds_order_dict = utils.convert_mappinglist_to_orderdict(extractors) + extract_binds_order_dict = utils.ensure_mapping_format(extractors) for key, field in extract_binds_order_dict.items(): extracted_variables_mapping[key] = self.extract_field(field) diff --git a/httprunner/utils.py b/httprunner/utils.py index 78e14c64..4dbc6280 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -177,32 +177,6 @@ def lower_test_dict_keys(test_dict): return test_dict -def convert_mappinglist_to_orderdict(mapping_list): - """ convert mapping list to ordered dict - - Args: - mapping_list (list): - [ - {"a": 1}, - {"b": 2} - ] - - Returns: - OrderedDict: converted mapping in OrderedDict - OrderDict( - { - "a": 1, - "b": 2 - } - ) - - """ - ordered_dict = OrderedDict() - for map_dict in mapping_list: - ordered_dict.update(map_dict) - - return ordered_dict - def deepcopy_dict(data): """ deepcopy dict data, ignore file object (_io.BufferedReader) @@ -288,7 +262,9 @@ def ensure_mapping_format(variables): """ if isinstance(variables, list): - variables_ordered_dict = convert_mappinglist_to_orderdict(variables) + variables_ordered_dict = OrderedDict() + for map_dict in variables: + variables_ordered_dict.update(map_dict) elif isinstance(variables, (OrderedDict, dict)): variables_ordered_dict = variables else: diff --git a/tests/test_utils.py b/tests/test_utils.py index 963e56c5..72f26730 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -206,12 +206,12 @@ class TestUtils(ApiServerUnittest): new_request_dict = utils.lower_dict_keys(request_dict) self.assertEqual(None, request_dict) - def test_convert_to_order_dict(self): + def test_ensure_mapping_format(self): map_list = [ {"a": 1}, {"b": 2} ] - ordered_dict = utils.convert_mappinglist_to_orderdict(map_list) + ordered_dict = utils.ensure_mapping_format(map_list) self.assertIsInstance(ordered_dict, dict) self.assertIn("a", ordered_dict) @@ -240,7 +240,7 @@ class TestUtils(ApiServerUnittest): {"a": 1}, {"b": 2} ] - ordered_dict = utils.convert_mappinglist_to_orderdict(map_list) + ordered_dict = utils.ensure_mapping_format(map_list) override_mapping = {"a": 3, "c": 4} new_dict = utils.update_ordered_dict(ordered_dict, override_mapping) self.assertEqual(3, new_dict["a"]) From badb473520c6eafa30719cdc1ff6383d9eae3d5c Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 14 Nov 2018 22:11:14 +0800 Subject: [PATCH 011/113] set_os_environ before parsing testcase, unset_os_environ after parsing testcase --- httprunner/loader.py | 1 - httprunner/parser.py | 7 +++++++ httprunner/utils.py | 10 +++++++++- tests/debugtalk.py | 1 - tests/test_loader.py | 1 - 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index 700f7582..d1e0d24f 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -179,7 +179,6 @@ def load_dot_env_file(dot_env_path): env_variables_mapping[variable.strip()] = value.strip() - utils.set_os_environ(env_variables_mapping) return env_variables_mapping diff --git a/httprunner/parser.py b/httprunner/parser.py index 481ddc9d..3d9f8a7d 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -618,6 +618,10 @@ def parse_tests(testcases, variables_mapping=None): } ) + env_mapping = project_mapping["env"] + # set OS environment variables + utils.set_os_environ(env_mapping) + # parse config parameters config_parameters = testcase_config.pop("parameters", []) cartesian_product_parameters_list = parse_parameters( @@ -673,4 +677,7 @@ def parse_tests(testcases, variables_mapping=None): testcase_dict["config"]["functions"] = project_mapping["debugtalk"]["functions"] parsed_testcases_list.append(testcase_dict) + # unset OS environment variables + utils.unset_os_environ(env_mapping) + return parsed_testcases_list diff --git a/httprunner/utils.py b/httprunner/utils.py index 4dbc6280..5ba9a161 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -26,7 +26,15 @@ def set_os_environ(variables_mapping): """ for variable in variables_mapping: os.environ[variable] = variables_mapping[variable] - logger.log_debug("Loaded variable: {}".format(variable)) + logger.log_debug("Set OS environment variable: {}".format(variable)) + + +def unset_os_environ(variables_mapping): + """ set variables mapping to os.environ + """ + for variable in variables_mapping: + os.environ.pop(variable) + logger.log_debug("Unset OS environment variable: {}".format(variable)) def query_json(json_content, query, delimiter='.'): diff --git a/tests/debugtalk.py b/tests/debugtalk.py index b44239de..4e22fa0f 100644 --- a/tests/debugtalk.py +++ b/tests/debugtalk.py @@ -7,7 +7,6 @@ import time from tests.api_server import HTTPBIN_SERVER, SECRET_KEY, gen_md5, get_sign BASE_URL = "http://127.0.0.1:5000" -UserName = os.environ['UserName'] demo_default_request = { diff --git a/tests/test_loader.py b/tests/test_loader.py index 104a855c..fc126873 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -565,7 +565,6 @@ class TestSuiteLoader(unittest.TestCase): path = os.path.join( os.getcwd(), 'tests/data/demo_locust.yml') locust_tests = loader.load_locust_tests(path) - self.assertEqual(locust_tests["config"]["variables"]["UserName"], "debugtalk") self.assertEqual(len(locust_tests["tests"]), 10) self.assertEqual(locust_tests["tests"][0][0]["name"], "index") self.assertEqual(locust_tests["tests"][9][0]["name"], "user-agent") From 626612061ee235c808043b72fcdec36502a1c4a7 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 15 Nov 2018 14:48:44 +0800 Subject: [PATCH 012/113] remove debugtalk.py variables mechanism --- httprunner/context.py | 4 +- httprunner/loader.py | 129 +++++++------------------ httprunner/parser.py | 63 ++++-------- tests/api/basic.yml | 2 +- tests/data/bugfix_parameters.yml | 27 ++++++ tests/data/demo_parameters.yml | 2 +- tests/data/demo_testcase.yml | 2 +- tests/data/demo_testcase_functions.yml | 2 +- tests/data/demo_testcase_layer.yml | 2 +- tests/data/demo_testcase_variables.yml | 2 +- tests/debugtalk.py | 18 ++-- tests/httpbin/hooks.yml | 4 +- tests/httpbin/load_image.yml | 2 +- tests/httpbin/upload.yml | 2 +- tests/test_api.py | 2 +- tests/test_context.py | 6 +- tests/test_loader.py | 65 ++----------- tests/test_parser.py | 14 +-- tests/test_response.py | 3 +- tests/test_runner.py | 18 ++-- tests/test_utils.py | 3 +- 21 files changed, 135 insertions(+), 237 deletions(-) create mode 100644 tests/data/bugfix_parameters.yml diff --git a/httprunner/context.py b/httprunner/context.py index 55bd1da6..89f76800 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -13,10 +13,12 @@ class Context(object): def __init__(self, variables=None, functions=None): """ init Context with testcase variables and functions. """ + variables = variables or {} + functions = functions or {} # testcase level context ## TESTCASE_SHARED_VARIABLES_MAPPING and TESTCASE_SHARED_FUNCTIONS_MAPPING are unchangeable. self.TESTCASE_SHARED_VARIABLES_MAPPING = utils.ensure_mapping_format(variables) - self.TESTCASE_SHARED_FUNCTIONS_MAPPING = functions or OrderedDict() + self.TESTCASE_SHARED_FUNCTIONS_MAPPING = functions # testcase level request, will not change self.TESTCASE_SHARED_REQUEST_MAPPING = {} diff --git a/httprunner/loader.py b/httprunner/loader.py index d1e0d24f..1c138150 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -8,7 +8,7 @@ import os import sys import yaml -from httprunner import built_in, exceptions, logger, parser, utils, validator +from httprunner import exceptions, logger, parser, utils, validator from httprunner.compat import OrderedDict ############################################################################### @@ -179,6 +179,7 @@ def load_dot_env_file(dot_env_path): env_variables_mapping[variable.strip()] = value.strip() + utils.set_os_environ(env_variables_mapping) return env_variables_mapping @@ -219,96 +220,58 @@ def locate_file(start_path, file_name): ## debugtalk.py module loader ############################################################################### -def load_python_module(module): - """ load python module. +def load_module_functions(module): + """ load python module functions. Args: module: python module Returns: - dict: variables and functions mapping for specified python module + dict: functions mapping for specified python module { - "variables": {}, - "functions": {} + "func1_name": func1, + "func2_name": func2 } """ - # TODO (2.0): remove variables from debugtalk.py - debugtalk_module = { - "variables": {}, - "functions": {} - } + module_functions = {} for name, item in vars(module).items(): if validator.is_function((name, item)): - debugtalk_module["functions"][name] = item - elif validator.is_variable((name, item)): - if isinstance(item, tuple): - continue - debugtalk_module["variables"][name] = item - else: - pass + module_functions[name] = item - return debugtalk_module + return module_functions -def load_builtin_module(): - """ load built_in module +def load_builtin_functions(): + """ load built_in module functions """ - built_in_module = load_python_module(built_in) - return built_in_module + from httprunner import built_in + return load_module_functions(built_in) -def load_debugtalk_module(): - """ load project debugtalk.py module +def load_debugtalk_functions(debugtalk_path): + """ load project debugtalk.py module functions debugtalk.py should be located in project working directory. + Args: + debugtalk_path(str): debugtalk.py path + Returns: - dict: debugtalk module mapping + dict: debugtalk module functions mapping { - "variables": {}, - "functions": {} + "func1_name": func1, + "func2_name": func2 } """ + if not debugtalk_path: + return {} + # load debugtalk.py module imported_module = importlib.import_module("debugtalk") - debugtalk_module = load_python_module(imported_module) - return debugtalk_module - - -def get_module_item(module_mapping, item_type, item_name): - """ get expected function or variable from module mapping. - - Args: - module_mapping(dict): module mapping with variables and functions. - - { - "variables": {}, - "functions": {} - } - - item_type(str): "functions" or "variables" - item_name(str): function name or variable name - - Returns: - object: specified variable or function object. - - Raises: - exceptions.FunctionNotFound: If specified function not found in module mapping - exceptions.VariableNotFound: If specified variable not found in module mapping - - """ - try: - return module_mapping[item_type][item_name] - except KeyError: - err_msg = "{} not found in debugtalk.py module!\n".format(item_name) - err_msg += "module mapping: {}".format(module_mapping) - if item_type == "functions": - raise exceptions.FunctionNotFound(err_msg) - else: - raise exceptions.VariableNotFound(err_msg) + return load_module_functions(imported_module) ############################################################################### @@ -899,7 +862,7 @@ def load_project_tests(test_path, dot_env_path=None): dot_env_path (str): specified .env file path Returns: - dict: project loaded api/testcases definitions, environments and debugtalk.py module. + dict: project loaded api/testcases definitions, environments and debugtalk.py functions. """ project_mapping = {} @@ -924,13 +887,7 @@ def load_project_tests(test_path, dot_env_path=None): project_mapping["env"] = {} # load debugtalk.py - if debugtalk_path: - project_mapping["debugtalk"] = load_debugtalk_module() - else: - project_mapping["debugtalk"] = { - "variables": {}, - "functions": {} - } + project_mapping["functions"] = load_debugtalk_functions(debugtalk_path) project_mapping["def-api"] = load_api_folder(os.path.join(project_working_directory, "api")) # TODO: replace suite with testcases @@ -960,10 +917,7 @@ def load_tests(path, dot_env_path=None): "variables": [], # optional "request": {} # optional "refs": { - "debugtalk": { - "variables": {}, - "functions": {} - }, + "functions": {}, "env": {}, "def-api": {}, "def-testcase": {} @@ -1049,10 +1003,7 @@ def load_locust_tests(path, dot_env_path=None): raw_testcase = load_file(path) project_mapping = load_project_tests(path, dot_env_path) - config = { - "variables": project_mapping["debugtalk"]["variables"], - "functions": project_mapping["debugtalk"]["functions"] - } + config = {} tests = [] for item in raw_testcase: key, test_block = item.popitem() @@ -1067,31 +1018,25 @@ def load_locust_tests(path, dot_env_path=None): # parse config variables raw_config_variables = config.get("variables", []) - parsed_config_variables = parser.parse_data( - raw_config_variables, - project_mapping["debugtalk"]["variables"], - project_mapping["debugtalk"]["functions"] - ) - # priority: passed in > debugtalk.py > parameters > variables - # override variables mapping with parameters mapping - config_variables = utils.override_mapping_list( - parsed_config_variables, {}) - # merge debugtalk.py module variables - config_variables.update(project_mapping["debugtalk"]["variables"]) + config_variables = parser.parse_data( + raw_config_variables, + {}, + project_mapping["functions"] + ) # parse config name config["name"] = parser.parse_data( config.get("name", ""), config_variables, - project_mapping["debugtalk"]["functions"] + project_mapping["functions"] ) # parse config request config["request"] = parser.parse_data( config.get("request", {}), config_variables, - project_mapping["debugtalk"]["functions"] + project_mapping["functions"] ) return { diff --git a/httprunner/parser.py b/httprunner/parser.py index 3d9f8a7d..dd3b1b99 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -325,34 +325,6 @@ def parse_parameters(parameters, variables_mapping, functions_mapping): ## parse content with variables and functions mapping ############################################################################### -def get_builtin_item(item_type, item_name): - """ - - Args: - item_type (enum): "variables" or "functions" - item_name (str): variable name or function name - - Returns: - variable or function with the name of item_name - - """ - # override built_in module with debugtalk.py module - from httprunner import loader - built_in_module = loader.load_builtin_module() - - if item_type == "variables": - try: - return built_in_module["variables"][item_name] - except KeyError: - raise exceptions.VariableNotFound("{} is not found.".format(item_name)) - else: - # item_type == "functions": - try: - return built_in_module["functions"][item_name] - except KeyError: - raise exceptions.FunctionNotFound("{} is not found.".format(item_name)) - - def get_mapping_variable(variable_name, variables_mapping): """ get variable from variables_mapping. @@ -367,10 +339,10 @@ def get_mapping_variable(variable_name, variables_mapping): exceptions.VariableNotFound: variable is not found. """ - if variable_name in variables_mapping: + try: return variables_mapping[variable_name] - else: - return get_builtin_item("variables", variable_name) + except KeyError: + raise exceptions.VariableNotFound("{} is not found.".format(variable_name)) def get_mapping_function(function_name, functions_mapping): @@ -392,12 +364,15 @@ def get_mapping_function(function_name, functions_mapping): return functions_mapping[function_name] try: - return get_builtin_item("functions", function_name) - except exceptions.FunctionNotFound: + # check if HttpRunner builtin functions + from httprunner import loader + built_in_functions = loader.load_builtin_functions() + return built_in_functions[function_name] + except KeyError: pass try: - # check if builtin functions + # check if Python builtin functions item_func = eval(function_name) if callable(item_func): # is builtin function @@ -608,10 +583,7 @@ def parse_tests(testcases, variables_mapping=None): project_mapping = testcase_config.pop( "refs", { - "debugtalk": { - "variables": {}, - "functions": {} - }, + "functions": {}, "env": {}, "def-api": {}, "def-testcase": {} @@ -626,8 +598,8 @@ def parse_tests(testcases, variables_mapping=None): config_parameters = testcase_config.pop("parameters", []) cartesian_product_parameters_list = parse_parameters( config_parameters, - project_mapping["debugtalk"]["variables"], - project_mapping["debugtalk"]["functions"] + {}, + project_mapping["functions"] ) or [{}] for parameter_mapping in cartesian_product_parameters_list: @@ -637,10 +609,9 @@ def parse_tests(testcases, variables_mapping=None): raw_config_variables = config.get("variables", []) raw_config_variables_mapping = utils.ensure_mapping_format(raw_config_variables) - # priority: passed in > .env > debugtalk.py > parameters > variables + # priority: passed in > parameters > variables config_variables = utils.deepcopy_dict(parameter_mapping) - config_variables.update(project_mapping["debugtalk"]["variables"]) config_variables.update(variables_mapping) for key, value in raw_config_variables_mapping.items(): @@ -653,7 +624,7 @@ def parse_tests(testcases, variables_mapping=None): parsed_value = parse_data( value, config_variables, - project_mapping["debugtalk"]["functions"] + project_mapping["functions"] ) config_variables[key] = parsed_value @@ -663,18 +634,18 @@ def parse_tests(testcases, variables_mapping=None): testcase_dict["config"]["name"] = parse_data( testcase_dict["config"].get("name", ""), config_variables, - project_mapping["debugtalk"]["functions"] + project_mapping["functions"] ) # parse config request testcase_dict["config"]["request"] = parse_data( testcase_dict["config"].get("request", {}), config_variables, - project_mapping["debugtalk"]["functions"] + project_mapping["functions"] ) # put loaded project functions to config - testcase_dict["config"]["functions"] = project_mapping["debugtalk"]["functions"] + testcase_dict["config"]["functions"] = project_mapping["functions"] parsed_testcases_list.append(testcase_dict) # unset OS environment variables diff --git a/tests/api/basic.yml b/tests/api/basic.yml index 75d3c236..41ea3327 100644 --- a/tests/api/basic.yml +++ b/tests/api/basic.yml @@ -94,4 +94,4 @@ - ${teardown_hook_sleep_N_secs($response, $n_secs)} validate: - eq: ["status_code", 200] - - contained_by: [content.headers.Host, $HTTPBIN_SERVER] + - contained_by: [content.headers.Host, "${get_httpbin_server()}"] diff --git a/tests/data/bugfix_parameters.yml b/tests/data/bugfix_parameters.yml new file mode 100644 index 00000000..e81dee42 --- /dev/null +++ b/tests/data/bugfix_parameters.yml @@ -0,0 +1,27 @@ +# fix #258 +- config: + name: "user management testcase." + variables: + - user_agent: 'iOS/10.3' + - device_sn: ${gen_random_string(15)} + - os_platform: 'ios' + - app_version: '2.8.6' + parameters: + - user_agent: ['iOS', 'android'] + request: + base_url: ${get_base_url()} + headers: + Content-Type: application/json + device_sn: $device_sn + output: + - token + +- test: + name: get token with $user_agent, $app_version + api: get_token($user_agent, $device_sn, $os_platform, $app_version) + extract: + - token: content.token + validate: + - "eq": ["status_code", 200] + - "len_eq": ["content.token", 16] + - "contains": [{"a": 1, "b": 2}, "b"] diff --git a/tests/data/demo_parameters.yml b/tests/data/demo_parameters.yml index 667cfd50..4b8eb66c 100644 --- a/tests/data/demo_parameters.yml +++ b/tests/data/demo_parameters.yml @@ -10,7 +10,7 @@ - os_platform: 'ios' - app_version: 2.8.5 request: - base_url: $BASE_URL + base_url: ${get_base_url()} headers: Content-Type: application/json device_sn: $device_sn diff --git a/tests/data/demo_testcase.yml b/tests/data/demo_testcase.yml index e16eea4d..595185ef 100644 --- a/tests/data/demo_testcase.yml +++ b/tests/data/demo_testcase.yml @@ -8,7 +8,7 @@ - [11, 21] - [12, 22] - "app_version": "${gen_app_version()}" - request: $demo_default_request + request: ${get_default_request()} - test: name: testcase1-$var_a diff --git a/tests/data/demo_testcase_functions.yml b/tests/data/demo_testcase_functions.yml index 1ffda197..7d69d8be 100644 --- a/tests/data/demo_testcase_functions.yml +++ b/tests/data/demo_testcase_functions.yml @@ -6,7 +6,7 @@ - os_platform: 'ios' - app_version: '2.8.6' request: - base_url: $BASE_URL + base_url: ${get_base_url()} headers: Content-Type: application/json device_sn: $device_sn diff --git a/tests/data/demo_testcase_layer.yml b/tests/data/demo_testcase_layer.yml index 3fc74718..aed22861 100644 --- a/tests/data/demo_testcase_layer.yml +++ b/tests/data/demo_testcase_layer.yml @@ -6,7 +6,7 @@ - os_platform: 'ios' - app_version: '2.8.6' request: - base_url: $BASE_URL + base_url: ${get_base_url()} headers: Content-Type: application/json device_sn: $device_sn diff --git a/tests/data/demo_testcase_variables.yml b/tests/data/demo_testcase_variables.yml index 2ca76b28..2fe15ae2 100644 --- a/tests/data/demo_testcase_variables.yml +++ b/tests/data/demo_testcase_variables.yml @@ -3,7 +3,7 @@ variables: - device_sn: 'HZfFBh6tU59EdXJ' request: - base_url: $BASE_URL + base_url: ${get_base_url()} headers: Content-Type: application/json device_sn: $device_sn diff --git a/tests/debugtalk.py b/tests/debugtalk.py index 4e22fa0f..19b3e6e3 100644 --- a/tests/debugtalk.py +++ b/tests/debugtalk.py @@ -4,17 +4,23 @@ import random import string import time -from tests.api_server import HTTPBIN_SERVER, SECRET_KEY, gen_md5, get_sign +from tests.api_server import HTTPBIN_SERVER, gen_md5, get_sign BASE_URL = "http://127.0.0.1:5000" +def get_httpbin_server(): + return HTTPBIN_SERVER -demo_default_request = { - "base_url": "$BASE_URL", - "headers": { - "content-type": "application/json" +def get_base_url(): + return BASE_URL + +def get_default_request(): + return { + "base_url": BASE_URL, + "headers": { + "content-type": "application/json" + } } -} def sum_two(m, n): return m + n diff --git a/tests/httpbin/hooks.yml b/tests/httpbin/hooks.yml index 7152f799..b7e8e8ef 100644 --- a/tests/httpbin/hooks.yml +++ b/tests/httpbin/hooks.yml @@ -1,7 +1,7 @@ - config: name: basic test with httpbin request: - base_url: $HTTPBIN_SERVER + base_url: ${get_httpbin_server()} setup_hooks: - ${hook_print(setup)} teardown_hooks: @@ -19,7 +19,7 @@ - ${teardown_hook_sleep_N_secs($response, 1)} validate: - eq: ["status_code", 200] - - contained_by: [content.headers.Host, $HTTPBIN_SERVER] + - contained_by: [content.headers.Host, "${get_httpbin_server()}"] - test: name: alter response diff --git a/tests/httpbin/load_image.yml b/tests/httpbin/load_image.yml index 545a12e7..8ada7f21 100644 --- a/tests/httpbin/load_image.yml +++ b/tests/httpbin/load_image.yml @@ -1,7 +1,7 @@ - config: name: load images request: - base_url: $HTTPBIN_SERVER + base_url: ${get_httpbin_server()} - test: name: get png image diff --git a/tests/httpbin/upload.yml b/tests/httpbin/upload.yml index e604a5cb..c3750511 100644 --- a/tests/httpbin/upload.yml +++ b/tests/httpbin/upload.yml @@ -1,7 +1,7 @@ - config: name: test upload file with httpbin request: - base_url: $HTTPBIN_SERVER + base_url: ${get_httpbin_server()} - test: name: upload file diff --git a/tests/test_api.py b/tests/test_api.py index 7bf769a6..f4f780b1 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -312,7 +312,7 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertIn("token", summary["details"][0]["in_out"]["out"]) - self.assertGreater(len(summary["details"][0]["in_out"]["in"]), 7) + self.assertGreater(len(summary["details"][0]["in_out"]["in"]), 3) def test_run_testcase_with_parameters(self): testcase_file_path = os.path.join( diff --git a/tests/test_context.py b/tests/test_context.py index 72af3358..2e2dd192 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -10,11 +10,9 @@ class TestContext(ApiServerUnittest): def setUp(self): project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) - self.debugtalk_module = project_mapping["debugtalk"] - self.context = context.Context( - self.debugtalk_module["variables"], - self.debugtalk_module["functions"] + variables={"SECRET_KEY": "DebugTalk"}, + functions=project_mapping["functions"] ) testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml') self.testcases = loader.load_file(testcase_file_path) diff --git a/tests/test_loader.py b/tests/test_loader.py index fc126873..f6565e3d 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -191,57 +191,21 @@ class TestFileLoader(unittest.TestCase): class TestModuleLoader(unittest.TestCase): def test_filter_module_functions(self): - module_mapping = loader.load_python_module(loader) - functions_dict = module_mapping["functions"] - self.assertIn("load_python_module", functions_dict) - self.assertNotIn("is_py3", functions_dict) + module_functions = loader.load_module_functions(loader) + self.assertIn("load_module_functions", module_functions) + self.assertNotIn("is_py3", module_functions) def test_load_debugtalk_module(self): project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "httprunner")) - imported_module_items = project_mapping["debugtalk"] - self.assertNotIn("SECRET_KEY", imported_module_items["variables"]) - self.assertNotIn("alter_response", imported_module_items["functions"]) + self.assertNotIn("alter_response", project_mapping["functions"]) project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) - imported_module_items = project_mapping["debugtalk"] - self.assertEqual( - imported_module_items["variables"]["SECRET_KEY"], - "DebugTalk" - ) - self.assertIn("alter_response", imported_module_items["functions"]) + self.assertIn("alter_response", project_mapping["functions"]) - is_status_code_200 = imported_module_items["functions"]["is_status_code_200"] + is_status_code_200 = project_mapping["functions"]["is_status_code_200"] self.assertTrue(is_status_code_200(200)) self.assertFalse(is_status_code_200(500)) - def test_get_module_item_functions(self): - from httprunner import utils - module_mapping = loader.load_python_module(utils) - - get_uniform_comparator = loader.get_module_item( - module_mapping, "functions", "get_uniform_comparator") - self.assertTrue(validator.is_function(("get_uniform_comparator", get_uniform_comparator))) - self.assertEqual(get_uniform_comparator("=="), "equals") - - with self.assertRaises(exceptions.FunctionNotFound): - loader.get_module_item(module_mapping, "functions", "gen_md4") - - def test_get_module_item_variables(self): - dot_env_path = os.path.join( - os.getcwd(), "tests", ".env" - ) - loader.load_dot_env_file(dot_env_path) - - from tests import debugtalk - module_mapping = loader.load_python_module(debugtalk) - - SECRET_KEY = loader.get_module_item(module_mapping, "variables", "SECRET_KEY") - self.assertTrue(validator.is_variable(("SECRET_KEY", SECRET_KEY))) - self.assertEqual(SECRET_KEY, "DebugTalk") - - with self.assertRaises(exceptions.VariableNotFound): - loader.get_module_item(module_mapping, "variables", "SECRET_KEY2") - def test_locate_debugtalk_py(self): debugtalk_path = loader.locate_debugtalk_py("tests/data/demo_testcase.yml") self.assertEqual( @@ -268,12 +232,12 @@ class TestModuleLoader(unittest.TestCase): self.assertIsInstance(testcases, list) self.assertEqual( testcases[0]["config"]["request"], - '$demo_default_request' + '${get_default_request()}' ) self.assertEqual(testcases[0]["config"]["name"], '123$var_a') self.assertIn( "sum_two", - testcases[0]["config"]["refs"]["debugtalk"]["functions"] + testcases[0]["config"]["refs"]["functions"] ) @@ -427,22 +391,14 @@ class TestSuiteLoader(unittest.TestCase): testcases_list = loader.load_tests(path) self.assertEqual(len(testcases_list), 1) self.assertEqual(len(testcases_list[0]["teststeps"]), 3) - self.assertEqual( - testcases_list[0]["config"]["refs"]["debugtalk"]["variables"]["SECRET_KEY"], - "DebugTalk" - ) - self.assertIn("get_sign", testcases_list[0]["config"]["refs"]["debugtalk"]["functions"]) + self.assertIn("get_sign", testcases_list[0]["config"]["refs"]["functions"]) # relative file path path = 'tests/data/demo_testcase_hardcode.yml' testcases_list = loader.load_tests(path) self.assertEqual(len(testcases_list), 1) self.assertEqual(len(testcases_list[0]["teststeps"]), 3) - self.assertEqual( - testcases_list[0]["config"]["refs"]["debugtalk"]["variables"]["SECRET_KEY"], - "DebugTalk" - ) - self.assertIn("get_sign", testcases_list[0]["config"]["refs"]["debugtalk"]["functions"]) + self.assertIn("get_sign", testcases_list[0]["config"]["refs"]["functions"]) # list/set container with file(s) path = [ @@ -556,7 +512,6 @@ class TestSuiteLoader(unittest.TestCase): def test_load_project_tests(self): project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) - self.assertEqual(project_mapping["debugtalk"]["variables"]["SECRET_KEY"], "DebugTalk") self.assertIn("get_token", project_mapping["def-api"]) self.assertIn("setup_and_reset", project_mapping["def-testcase"]) self.assertEqual(project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH") diff --git a/tests/test_parser.py b/tests/test_parser.py index 21f5109b..7b1eeccb 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -389,11 +389,11 @@ class TestParser(unittest.TestCase): ) loader.load_dot_env_file(dot_env_path) from tests import debugtalk - debugtalk_module = loader.load_python_module(debugtalk) + functions = loader.load_module_functions(debugtalk) cartesian_product_parameters = parser.parse_parameters( parameters, - debugtalk_module["variables"], - debugtalk_module["functions"] + {}, + functions ) self.assertEqual( len(cartesian_product_parameters), @@ -424,7 +424,7 @@ class TestParser(unittest.TestCase): {"username-password": "${parameterize(tests/data/account.csv)}"} ] variables_mapping = {} - functions_mapping = project_mapping["debugtalk"]["functions"] + functions_mapping = project_mapping["functions"] testcase_path = os.path.join( os.getcwd(), "tests/data/demo_parameters.yml" @@ -445,11 +445,7 @@ class TestParser(unittest.TestCase): self.assertEqual(len(parsed_testcases), 2 * 2) self.assertEqual( parsed_testcases[0]["config"]["request"]["base_url"], - '$BASE_URL' - ) - self.assertEqual( - parsed_testcases[0]["config"]["variables"]["BASE_URL"], - 'http://127.0.0.1:5000' + "http://127.0.0.1:5000" ) self.assertIsInstance(parsed_testcases, list) self.assertEqual(parsed_testcases[0]["config"]["name"], '12311') diff --git a/tests/test_response.py b/tests/test_response.py index c54058ae..f73825f4 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -8,8 +8,7 @@ from tests.base import ApiServerUnittest class TestResponse(ApiServerUnittest): def setUp(self): - module_mapping = loader.load_python_module(built_in) - self.functions_mapping = module_mapping["functions"] + self.functions_mapping = loader.load_module_functions(built_in) def test_parse_response_object_json(self): url = "http://127.0.0.1:5000/api/users" diff --git a/tests/test_runner.py b/tests/test_runner.py index 51009470..a283de4f 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -11,10 +11,10 @@ class TestRunner(ApiServerUnittest): def setUp(self): project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) - self.debugtalk_module = project_mapping["debugtalk"] + self.debugtalk_functions = project_mapping["functions"] config_dict = { - "variables": self.debugtalk_module["variables"], - "functions": self.debugtalk_module["functions"] + "variables": {}, + "functions": self.debugtalk_functions } self.test_runner = runner.Runner(config_dict) self.reset_all() @@ -36,8 +36,8 @@ class TestRunner(ApiServerUnittest): testcases = loader.load_file(testcase_file_path) config_dict = { - "variables": self.debugtalk_module["variables"], - "functions": self.debugtalk_module["functions"] + "variables": {}, + "functions": self.debugtalk_functions } test_runner = runner.Runner(config_dict) @@ -81,8 +81,8 @@ class TestRunner(ApiServerUnittest): config_dict = { "name": "basic test with httpbin", - "variables": self.debugtalk_module["variables"], - "functions": self.debugtalk_module["functions"], + "variables": {}, + "functions": self.debugtalk_functions, "request": { "base_url": HTTPBIN_SERVER }, @@ -130,8 +130,8 @@ class TestRunner(ApiServerUnittest): def test_run_testcase_with_hooks_modify_request(self): config_dict = { "name": "basic test with httpbin", - "variables": self.debugtalk_module["variables"], - "functions": self.debugtalk_module["functions"], + "variables": {}, + "functions": self.debugtalk_functions, "request": { "base_url": HTTPBIN_SERVER } diff --git a/tests/test_utils.py b/tests/test_utils.py index 72f26730..d9c86183 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -101,8 +101,7 @@ class TestUtils(ApiServerUnittest): def current_validators(self): from httprunner import built_in - module_mapping = loader.load_python_module(built_in) - functions_mapping = module_mapping["functions"] + functions_mapping = loader.load_module_functions(built_in) functions_mapping["equals"](None, None) functions_mapping["equals"](1, 1) From 58fa2a4bf4de9abfa90dc45924df5eb350624082 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 15 Nov 2018 16:06:49 +0800 Subject: [PATCH 013/113] relocate functions related to debugtalk.py --- httprunner/loader.py | 59 ++++++++++++++++++++++++-------------------- tests/test_loader.py | 31 ++++++++++++----------- 2 files changed, 49 insertions(+), 41 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index 1c138150..1841055f 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -184,7 +184,7 @@ def load_dot_env_file(dot_env_path): def locate_file(start_path, file_name): - """ locate filename and return file path. + """ locate filename and return absolute file path. searching will be recursive upward until current working directory. Args: @@ -206,7 +206,7 @@ def locate_file(start_path, file_name): file_path = os.path.join(start_dir_path, file_name) if os.path.isfile(file_path): - return file_path + return os.path.abspath(file_path) # current working directory if os.path.abspath(start_dir_path) in [os.getcwd(), os.path.abspath(os.sep)]: @@ -251,13 +251,10 @@ def load_builtin_functions(): return load_module_functions(built_in) -def load_debugtalk_functions(debugtalk_path): +def load_debugtalk_functions(): """ load project debugtalk.py module functions debugtalk.py should be located in project working directory. - Args: - debugtalk_path(str): debugtalk.py path - Returns: dict: debugtalk module functions mapping { @@ -266,9 +263,6 @@ def load_debugtalk_functions(debugtalk_path): } """ - if not debugtalk_path: - return {} - # load debugtalk.py module imported_module = importlib.import_module("debugtalk") return load_module_functions(imported_module) @@ -839,18 +833,40 @@ def load_test_folder(test_folder_path): return test_definition_mapping -def locate_debugtalk_py(start_path): - """ locate debugtalk.py file. +def load_debugtalk_py(start_path): + """ locate debugtalk.py file and returns PWD and debugtalk.py functions. Args: start_path (str): start locating path, maybe testcase file path or directory path + Returns: + tuple: (project_working_directory, debugtalk_functions) + """ try: + # locate debugtalk.py file. debugtalk_path = locate_file(start_path, "debugtalk.py") - return os.path.abspath(debugtalk_path) + + # The folder contains debugtalk.py will be treated as PWD. + project_working_directory = os.path.dirname(debugtalk_path) + + # add PWD to sys.path + sys.path.insert(0, project_working_directory) + + # load debugtalk.py functions + debugtalk_functions = load_debugtalk_functions() + except exceptions.FileNotFound: - return None + + # debugtalk.py not found, use os.getcwd() as PWD. + project_working_directory = os.getcwd() + + # add PWD to sys.path + sys.path.insert(0, project_working_directory) + + debugtalk_functions = {} + + return project_working_directory, debugtalk_functions def load_project_tests(test_path, dot_env_path=None): @@ -867,17 +883,9 @@ def load_project_tests(test_path, dot_env_path=None): """ project_mapping = {} - debugtalk_path = locate_debugtalk_py(test_path) - # locate PWD with debugtalk.py path - if debugtalk_path: - # The folder contains debugtalk.py will be treated as PWD. - project_working_directory = os.path.dirname(debugtalk_path) - else: - # debugtalk.py is not found, use os.getcwd() as PWD. - project_working_directory = os.getcwd() - - # add PWD to sys.path - sys.path.insert(0, project_working_directory) + # locate PWD and load debugtalk.py functions + project_working_directory, debugtalk_functions = load_debugtalk_py(test_path) + project_mapping["functions"] = debugtalk_functions # load .env dot_env_path = dot_env_path or os.path.join(project_working_directory, ".env") @@ -886,9 +894,6 @@ def load_project_tests(test_path, dot_env_path=None): else: project_mapping["env"] = {} - # load debugtalk.py - project_mapping["functions"] = load_debugtalk_functions(debugtalk_path) - project_mapping["def-api"] = load_api_folder(os.path.join(project_working_directory, "api")) # TODO: replace suite with testcases project_mapping["def-testcase"] = load_test_folder(os.path.join(project_working_directory, "suite")) diff --git a/tests/test_loader.py b/tests/test_loader.py index f6565e3d..8f8fd0c8 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -172,19 +172,19 @@ class TestFileLoader(unittest.TestCase): ) self.assertEqual( loader.locate_file("tests/", "debugtalk.py"), - "tests/debugtalk.py" + os.path.join(os.getcwd(), "tests", "debugtalk.py") ) self.assertEqual( loader.locate_file("tests", "debugtalk.py"), - "tests/debugtalk.py" + os.path.join(os.getcwd(), "tests", "debugtalk.py") ) self.assertEqual( loader.locate_file("tests/base.py", "debugtalk.py"), - "tests/debugtalk.py" + os.path.join(os.getcwd(), "tests", "debugtalk.py") ) self.assertEqual( loader.locate_file("tests/data/demo_testcase.yml", "debugtalk.py"), - "tests/debugtalk.py" + os.path.join(os.getcwd(), "tests", "debugtalk.py") ) @@ -206,24 +206,27 @@ class TestModuleLoader(unittest.TestCase): self.assertTrue(is_status_code_200(200)) self.assertFalse(is_status_code_200(500)) - def test_locate_debugtalk_py(self): - debugtalk_path = loader.locate_debugtalk_py("tests/data/demo_testcase.yml") + def test_load_debugtalk_py(self): + project_working_directory, debugtalk_functions = loader.load_debugtalk_py("tests/data/demo_testcase.yml") self.assertEqual( - debugtalk_path, - os.path.join(os.getcwd(), "tests", "debugtalk.py") + project_working_directory, + os.path.join(os.getcwd(), "tests") ) + self.assertIn("gen_md5", debugtalk_functions) - debugtalk_path = loader.locate_debugtalk_py("tests/base.py") + project_working_directory, debugtalk_functions = loader.load_debugtalk_py("tests/base.py") self.assertEqual( - debugtalk_path, - os.path.join(os.getcwd(), "tests", "debugtalk.py") + project_working_directory, + os.path.join(os.getcwd(), "tests") ) + self.assertIn("gen_md5", debugtalk_functions) - debugtalk_path = loader.locate_debugtalk_py("httprunner/__init__.py") + project_working_directory, debugtalk_functions = loader.load_debugtalk_py("httprunner/__init__.py") self.assertEqual( - debugtalk_path, - None + project_working_directory, + os.getcwd() ) + self.assertEqual(debugtalk_functions, {}) def test_load_tests(self): testcase_file_path = os.path.join( From 0e54c4aec90176519d8fa59202d01a524a92c667 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 15 Nov 2018 19:37:50 +0800 Subject: [PATCH 014/113] relocate functions related to .env --- httprunner/loader.py | 9 ++++----- tests/test_loader.py | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index 1841055f..acb65fbc 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -163,10 +163,11 @@ def load_dot_env_file(dot_env_path): """ if not os.path.isfile(dot_env_path): - raise exceptions.FileNotFound(".env file path is not exist.") + return {} logger.log_info("Loading environment variables from {}".format(dot_env_path)) env_variables_mapping = {} + with io.open(dot_env_path, 'r', encoding='utf-8') as fp: for line in fp: # maxsplit=1 @@ -889,11 +890,9 @@ def load_project_tests(test_path, dot_env_path=None): # load .env dot_env_path = dot_env_path or os.path.join(project_working_directory, ".env") - if os.path.isfile(dot_env_path): - project_mapping["env"] = load_dot_env_file(dot_env_path) - else: - project_mapping["env"] = {} + project_mapping["env"] = load_dot_env_file(dot_env_path) + # load api/testcases project_mapping["def-api"] = load_api_folder(os.path.join(project_working_directory, "api")) # TODO: replace suite with testcases project_mapping["def-testcase"] = load_test_folder(os.path.join(project_working_directory, "suite")) diff --git a/tests/test_loader.py b/tests/test_loader.py index 8f8fd0c8..91f7850e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -153,8 +153,8 @@ class TestFileLoader(unittest.TestCase): dot_env_path = os.path.join( os.getcwd(), "tests", "data", ) - with self.assertRaises(exceptions.FileNotFound): - loader.load_dot_env_file(dot_env_path) + env_variables_mapping = loader.load_dot_env_file(dot_env_path) + self.assertEqual(env_variables_mapping, {}) def test_locate_file(self): with self.assertRaises(exceptions.FileNotFound): From a116c651f90150e25f5173efddd6ebef415af692 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 15 Nov 2018 19:58:26 +0800 Subject: [PATCH 015/113] remove unused utils functions --- httprunner/utils.py | 74 --------------------------------------------- tests/test_utils.py | 45 --------------------------- 2 files changed, 119 deletions(-) diff --git a/httprunner/utils.py b/httprunner/utils.py index 5ba9a161..d1ccc4a8 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -13,14 +13,6 @@ from httprunner import exceptions, logger from httprunner.compat import OrderedDict, basestring, is_py2 -def remove_prefix(text, prefix): - """ remove prefix from text - """ - if text.startswith(prefix): - return text[len(prefix):] - return text - - def set_os_environ(variables_mapping): """ set variables mapping to os.environ """ @@ -222,30 +214,6 @@ def deepcopy_dict(data): return copied_data -def update_ordered_dict(ordered_dict, override_mapping): - """ override ordered_dict with new mapping. - - Args: - ordered_dict (OrderDict): original ordered dict - override_mapping (dict): new variables mapping - - Returns: - OrderDict: new overrided variables mapping. - - Examples: - >>> ordered_dict = OrderDict({"a": 1, "b": 2}) - >>> override_mapping = {"a": 3, "c": 4} - >>> update_ordered_dict(ordered_dict, override_mapping) - OrderDict({"a": 3, "b": 2, "c": 4}) - - """ - new_ordered_dict = copy.copy(ordered_dict) - for var, value in override_mapping.items(): - new_ordered_dict.update({var: value}) - - return new_ordered_dict - - def ensure_mapping_format(variables): """ ensure variables are in mapping format. @@ -281,48 +249,6 @@ def ensure_mapping_format(variables): return variables_ordered_dict -def override_mapping_list(variables, new_mapping): - """ override variables with new mapping. - - Args: - variables (list): variables list - [ - {"var_a": 1}, - {"var_b": "world"} - ] - new_mapping (dict): overrided variables mapping - { - "var_a": "hello" - } - - Returns: - OrderedDict: overrided variables mapping. - - Examples: - >>> variables = [ - {"var_a": 1}, - {"var_b": "world"} - ] - >>> new_mapping = { - "var_a": "hello" - } - >>> override_mapping_list(variables, new_mapping) - OrderedDict( - { - "var_a": "hello", - "var_b": "world" - } - ) - - """ - variables_ordered_dict = ensure_mapping_format(variables) - - return update_ordered_dict( - variables_ordered_dict, - new_mapping - ) - - def get_testcase_io(testcase): """ get testcase input(variables) and output. diff --git a/tests/test_utils.py b/tests/test_utils.py index d9c86183..4965fd71 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,14 +9,6 @@ from tests.base import ApiServerUnittest class TestUtils(ApiServerUnittest): - def test_remove_prefix(self): - full_url = "http://debugtalk.com/post/123" - prefix = "http://debugtalk.com" - self.assertEqual( - utils.remove_prefix(full_url, prefix), - "/post/123" - ) - def test_set_os_environ(self): self.assertNotIn("abc", os.environ) variables_mapping = { @@ -234,43 +226,6 @@ class TestUtils(ApiServerUnittest): self.assertEqual(id(new_data["c"]), id(data["c"])) # self.assertEqual(id(new_data["d"]), id(data["d"])) - def test_update_ordered_dict(self): - map_list = [ - {"a": 1}, - {"b": 2} - ] - ordered_dict = utils.ensure_mapping_format(map_list) - override_mapping = {"a": 3, "c": 4} - new_dict = utils.update_ordered_dict(ordered_dict, override_mapping) - self.assertEqual(3, new_dict["a"]) - self.assertEqual(4, new_dict["c"]) - - def test_override_variables_binds(self): - map_list = [ - {"a": 1}, - {"b": 2} - ] - override_mapping = {"a": 3, "c": 4} - new_dict = utils.override_mapping_list(map_list, override_mapping) - self.assertEqual(3, new_dict["a"]) - self.assertEqual(4, new_dict["c"]) - - map_list = OrderedDict( - { - "a": 1, - "b": 2 - } - ) - override_mapping = {"a": 3, "c": 4} - new_dict = utils.override_mapping_list(map_list, override_mapping) - self.assertEqual(3, new_dict["a"]) - self.assertEqual(4, new_dict["c"]) - - map_list = "invalid" - override_mapping = {"a": 3, "c": 4} - with self.assertRaises(exceptions.ParamsError): - utils.override_mapping_list(map_list, override_mapping) - def test_create_scaffold(self): project_name = "projectABC" utils.create_scaffold(project_name) From bb3dcb2756626ee3b78da33b75129d9919ee2e75 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 15 Nov 2018 20:05:57 +0800 Subject: [PATCH 016/113] refactor is_function --- httprunner/loader.py | 2 +- httprunner/validator.py | 5 ++--- tests/test_validator.py | 5 ++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index acb65fbc..d7738893 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -239,7 +239,7 @@ def load_module_functions(module): module_functions = {} for name, item in vars(module).items(): - if validator.is_function((name, item)): + if validator.is_function(item): module_functions[name] = item return module_functions diff --git a/httprunner/validator.py b/httprunner/validator.py index b4c8601e..54d7ffc9 100644 --- a/httprunner/validator.py +++ b/httprunner/validator.py @@ -105,10 +105,9 @@ def is_testcase_path(path): ############################################################################### -def is_function(tup): - """ Takes (name, object) tuple, returns True if it is a function. +def is_function(item): + """ Takes item object, returns True if it is a function. """ - name, item = tup return isinstance(item, types.FunctionType) diff --git a/tests/test_validator.py b/tests/test_validator.py index 49a291f6..f83d876f 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -50,6 +50,5 @@ class TestValidator(unittest.TestCase): def test_is_function(self): func = lambda x: x + 1 - self.assertTrue(validator.is_function(("func", func))) - - self.assertTrue(validator.is_function(("func", validator.is_testcase))) + self.assertTrue(validator.is_function(func)) + self.assertTrue(validator.is_function(validator.is_testcase)) From 822df7005a3b35475864fde255cf630d98c44e0a Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 15 Nov 2018 20:35:17 +0800 Subject: [PATCH 017/113] remove obsolete key words: variable_binds, extractors, extract_binds, validators --- httprunner/context.py | 1 - httprunner/loader.py | 12 ++++-------- httprunner/runner.py | 10 +++++----- tests/httpbin/upload.yml | 2 +- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/httprunner/context.py b/httprunner/context.py index 89f76800..45dd3b51 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -3,7 +3,6 @@ import copy from httprunner import exceptions, logger, parser, utils -from httprunner.compat import OrderedDict class Context(object): diff --git a/httprunner/loader.py b/httprunner/loader.py index d7738893..2c4d4515 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -507,20 +507,16 @@ def _extend_block(ref_block, def_block): ) # merge & override validators - def_validators = def_block.pop("validate", None) or def_block.pop("validators", []) - ref_validators = ref_block.pop("validate", None) or ref_block.pop("validators", []) + def_validators = def_block.pop("validate", []) + ref_validators = ref_block.pop("validate", []) extended_block["validate"] = _extend_validators( def_validators, ref_validators ) # merge & override extractors - def_extrators = def_block.pop("extract", None) \ - or def_block.pop("extractors", None) \ - or def_block.pop("extract_binds", []) - ref_extractors = ref_block.pop("extract", None) \ - or ref_block.pop("extractors", None) \ - or ref_block.pop("extract_binds", []) + def_extrators = def_block.pop("extract", []) + ref_extractors = ref_block.pop("extract", []) extended_block["extract"] = _extend_variables( def_extrators, ref_extractors diff --git a/httprunner/runner.py b/httprunner/runner.py index 89fb657f..bcdda8bd 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -11,7 +11,8 @@ from httprunner.context import Context class Runner(object): def __init__(self, config_dict=None, http_client_session=None): - """ + """ Each testcase is corresponding to one Runner() instance + and one Context() instance. """ self.http_client_session = http_client_session config_dict = config_dict or {} @@ -76,8 +77,7 @@ class Runner(object): test_dict = utils.lower_test_dict_keys(test_dict) self.context.init_context_variables(level) - variables = test_dict.get('variables') \ - or test_dict.get('variable_binds', OrderedDict()) + variables = test_dict.get('variables', OrderedDict()) self.context.update_context_variables(variables, level) request_config = test_dict.get('request', {}) @@ -162,8 +162,8 @@ class Runner(object): self._handle_skip_feature(teststep_dict) # prepare - extractors = teststep_dict.get("extract", []) or teststep_dict.get("extractors", []) - validators = teststep_dict.get("validate", []) or teststep_dict.get("validators", []) + extractors = teststep_dict.get("extract", []) + validators = teststep_dict.get("validate", []) parsed_request = self.init_test(teststep_dict, level="teststep") self.context.update_teststep_variables_mapping("request", parsed_request) diff --git a/tests/httpbin/upload.yml b/tests/httpbin/upload.yml index c3750511..38d64a93 100644 --- a/tests/httpbin/upload.yml +++ b/tests/httpbin/upload.yml @@ -5,7 +5,7 @@ - test: name: upload file - variable_binds: + variables: - field_name: "file" - file_path: "LICENSE" - file_type: "text/html" From a67216cbdd9d7c2512dfd8beacfbc0b533b4e540 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 15 Nov 2018 21:09:22 +0800 Subject: [PATCH 018/113] update parse_parameters --- httprunner/api.py | 19 ++----------------- httprunner/parser.py | 11 +++++++---- tests/test_parser.py | 16 ++++------------ 3 files changed, 13 insertions(+), 33 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index e5e5e97d..8bd47703 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -20,18 +20,6 @@ class HttpRunner(object): failfast (bool): False/True, stop the test run on the first error or failure. http_client_session (instance): requests.Session(), or locust.client.Session() instance. - Attributes: - project_mapping (dict): save project loaded api/testcases, environments and debugtalk.py module. - { - "debugtalk": { - "variables": {}, - "functions": {} - }, - "env": {}, - "def-api": {}, - "def-testcase": {} - } - """ self.exception_stage = "initialize HttpRunner()" self.http_client_session = kwargs.pop("http_client_session", None) @@ -47,7 +35,7 @@ class HttpRunner(object): testcases (list): parsed testcases list Returns: - tuple: unittest.TestSuite() + unittest.TestSuite() """ def _add_teststep(test_runner, config, teststep_dict): @@ -165,10 +153,7 @@ class HttpRunner(object): "variables": [], # optional "request": {} # optional "refs": { - "debugtalk": { - "variables": {}, - "functions": {} - }, + "functions": {}, "env": {}, "def-api": {}, "def-testcase": {} diff --git a/httprunner/parser.py b/httprunner/parser.py index dd3b1b99..d1ab3312 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -255,7 +255,7 @@ def substitute_variables(content, variables_mapping): return content -def parse_parameters(parameters, variables_mapping, functions_mapping): +def parse_parameters(parameters, variables_mapping=None, functions_mapping=None): """ parse parameters and generate cartesian product. Args: @@ -265,7 +265,7 @@ def parse_parameters(parameters, variables_mapping, functions_mapping): (2) call built-in parameterize function, "${parameterize(account.csv)}" (3) call custom function in debugtalk.py, "${gen_app_version()}" - variables_mapping (dict): variables mapping loaded from debugtalk.py + variables_mapping (dict): variables mapping loaded from testcase config functions_mapping (dict): functions mapping loaded from debugtalk.py Returns: @@ -280,7 +280,10 @@ def parse_parameters(parameters, variables_mapping, functions_mapping): >>> parse_parameters(parameters) """ + variables_mapping = variables_mapping or {} + functions_mapping = functions_mapping or {} parsed_parameters_list = [] + for parameter in parameters: parameter_name, parameter_content = list(parameter.items())[0] parameter_name_list = parameter_name.split("-") @@ -542,7 +545,7 @@ def parse_tests(testcases, variables_mapping=None): "config": { "name": "desc1", "path": "testcase1_path", - "variables": [], # optional + "variables": {}, # optional "request": {} # optional "refs": { "debugtalk": { @@ -598,7 +601,7 @@ def parse_tests(testcases, variables_mapping=None): config_parameters = testcase_config.pop("parameters", []) cartesian_product_parameters_list = parse_parameters( config_parameters, - {}, + testcase_config.get("variables", {}), project_mapping["functions"] ) or [{}] diff --git a/tests/test_parser.py b/tests/test_parser.py index 7b1eeccb..9dd7a6fd 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -364,8 +364,7 @@ class TestParser(unittest.TestCase): ] variables_mapping = {} functions_mapping = {} - cartesian_product_parameters = parser.parse_parameters( - parameters, variables_mapping, functions_mapping) + cartesian_product_parameters = parser.parse_parameters(parameters) self.assertEqual( len(cartesian_product_parameters), 3 * 2 @@ -389,11 +388,9 @@ class TestParser(unittest.TestCase): ) loader.load_dot_env_file(dot_env_path) from tests import debugtalk - functions = loader.load_module_functions(debugtalk) cartesian_product_parameters = parser.parse_parameters( parameters, - {}, - functions + functions_mapping=loader.load_module_functions(debugtalk) ) self.assertEqual( len(cartesian_product_parameters), @@ -405,11 +402,7 @@ class TestParser(unittest.TestCase): {"app_version": "${parameterize(tests/data/app_version.csv)}"}, {"username-password": "${parameterize(tests/data/account.csv)}"} ] - variables_mapping = {} - functions_mapping = {} - - cartesian_product_parameters = parser.parse_parameters( - parameters, variables_mapping, functions_mapping) + cartesian_product_parameters = parser.parse_parameters(parameters) self.assertEqual( len(cartesian_product_parameters), 2 * 3 @@ -424,13 +417,12 @@ class TestParser(unittest.TestCase): {"username-password": "${parameterize(tests/data/account.csv)}"} ] variables_mapping = {} - functions_mapping = project_mapping["functions"] testcase_path = os.path.join( os.getcwd(), "tests/data/demo_parameters.yml" ) cartesian_product_parameters = parser.parse_parameters( - parameters, variables_mapping, functions_mapping) + parameters, functions_mapping=project_mapping["functions"]) self.assertEqual( len(cartesian_product_parameters), 3 * 2 * 3 From 5013861f7426e7a167ed4279ce03d51ce99d548e Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 15 Nov 2018 21:49:05 +0800 Subject: [PATCH 019/113] refactor testcase config data structure --- httprunner/api.py | 12 +++++------- httprunner/loader.py | 16 +++++++--------- httprunner/parser.py | 35 +++++++++++------------------------ tests/test_api.py | 7 +++---- tests/test_loader.py | 6 +++--- 5 files changed, 29 insertions(+), 47 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index 8bd47703..b506d9f1 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -151,13 +151,11 @@ class HttpRunner(object): "name": "desc1", "path": "testcase1_path", "variables": [], # optional - "request": {} # optional - "refs": { - "functions": {}, - "env": {}, - "def-api": {}, - "def-testcase": {} - } + "request": {}, # optional + "functions": {}, + "env": {}, + "def-api": {}, + "def-testcase": {} }, "teststeps": [ # teststep data structure diff --git a/httprunner/loader.py b/httprunner/loader.py index 2c4d4515..a1535640 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -867,7 +867,7 @@ def load_debugtalk_py(start_path): def load_project_tests(test_path, dot_env_path=None): - """ load api, testcases, .env, builtin module and debugtalk.py. + """ load api, testcases, .env, debugtalk.py functions. api/testcases folder is relative to project_working_directory Args: @@ -915,13 +915,11 @@ def load_tests(path, dot_env_path=None): "name": "desc1", "path": "testcase1_path", "variables": [], # optional - "request": {} # optional - "refs": { - "functions": {}, - "env": {}, - "def-api": {}, - "def-testcase": {} - } + "request": {}, # optional + "functions": {}, + "env": {}, + "def-api": {}, + "def-testcase": {} }, "teststeps": [ # teststep data structure @@ -969,7 +967,7 @@ def load_tests(path, dot_env_path=None): project_mapping = load_project_tests(path, dot_env_path) testcase = _load_testcase(raw_testcase, project_mapping) testcase["config"]["path"] = path - testcase["config"]["refs"] = project_mapping + testcase["config"].update(project_mapping) testcases_list = [testcase] except exceptions.FileFormatError: testcases_list = [] diff --git a/httprunner/parser.py b/httprunner/parser.py index d1ab3312..ff68352a 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -547,15 +547,10 @@ def parse_tests(testcases, variables_mapping=None): "path": "testcase1_path", "variables": {}, # optional "request": {} # optional - "refs": { - "debugtalk": { - "variables": {}, - "functions": {} - }, - "env": {}, - "def-api": {}, - "def-testcase": {} - } + "functions": {}, + "env": {}, + "def-api": {}, + "def-testcase": {} }, "teststeps": [ # teststep data structure @@ -583,17 +578,9 @@ def parse_tests(testcases, variables_mapping=None): for testcase in testcases: testcase_config = testcase.setdefault("config", {}) - project_mapping = testcase_config.pop( - "refs", - { - "functions": {}, - "env": {}, - "def-api": {}, - "def-testcase": {} - } - ) + functions = testcase_config.get("functions", {}) - env_mapping = project_mapping["env"] + env_mapping = testcase_config.get("env", {}) # set OS environment variables utils.set_os_environ(env_mapping) @@ -602,7 +589,7 @@ def parse_tests(testcases, variables_mapping=None): cartesian_product_parameters_list = parse_parameters( config_parameters, testcase_config.get("variables", {}), - project_mapping["functions"] + functions ) or [{}] for parameter_mapping in cartesian_product_parameters_list: @@ -627,7 +614,7 @@ def parse_tests(testcases, variables_mapping=None): parsed_value = parse_data( value, config_variables, - project_mapping["functions"] + functions ) config_variables[key] = parsed_value @@ -637,18 +624,18 @@ def parse_tests(testcases, variables_mapping=None): testcase_dict["config"]["name"] = parse_data( testcase_dict["config"].get("name", ""), config_variables, - project_mapping["functions"] + functions ) # parse config request testcase_dict["config"]["request"] = parse_data( testcase_dict["config"].get("request", {}), config_variables, - project_mapping["functions"] + functions ) # put loaded project functions to config - testcase_dict["config"]["functions"] = project_mapping["functions"] + testcase_dict["config"]["functions"] = functions parsed_testcases_list.append(testcase_dict) # unset OS environment variables diff --git a/tests/test_api.py b/tests/test_api.py index f4f780b1..85a3d86a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -165,12 +165,11 @@ class TestHttpRunner(ApiServerUnittest): self.assertLess(end_time - start_time, 60) def test_run_httprunner_with_teardown_hooks_alter_response(self): + config = {"name": "test teardown hooks"} + config.update(loader.load_project_tests("tests")) testcases = [ { - "config": { - "name": "test teardown hooks", - "refs": loader.load_project_tests("tests") - }, + "config": config, "teststeps": [ { "name": "test teardown hooks", diff --git a/tests/test_loader.py b/tests/test_loader.py index 91f7850e..2c6dcc49 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -240,7 +240,7 @@ class TestModuleLoader(unittest.TestCase): self.assertEqual(testcases[0]["config"]["name"], '123$var_a') self.assertIn( "sum_two", - testcases[0]["config"]["refs"]["functions"] + testcases[0]["config"]["functions"] ) @@ -394,14 +394,14 @@ class TestSuiteLoader(unittest.TestCase): testcases_list = loader.load_tests(path) self.assertEqual(len(testcases_list), 1) self.assertEqual(len(testcases_list[0]["teststeps"]), 3) - self.assertIn("get_sign", testcases_list[0]["config"]["refs"]["functions"]) + self.assertIn("get_sign", testcases_list[0]["config"]["functions"]) # relative file path path = 'tests/data/demo_testcase_hardcode.yml' testcases_list = loader.load_tests(path) self.assertEqual(len(testcases_list), 1) self.assertEqual(len(testcases_list[0]["teststeps"]), 3) - self.assertIn("get_sign", testcases_list[0]["config"]["refs"]["functions"]) + self.assertIn("get_sign", testcases_list[0]["config"]["functions"]) # list/set container with file(s) path = [ From e8d7f8e7c5189ab6a69462db9a0d75fa1c4cd259 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 16 Nov 2018 12:29:02 +0800 Subject: [PATCH 020/113] .env variables should be referenced with builtin ENV/environ function --- httprunner/exceptions.py | 3 +++ httprunner/parser.py | 4 +++- httprunner/utils.py | 19 +++++++++++++++++++ tests/data/demo_testcase.yml | 1 + tests/test_loader.py | 1 - tests/test_parser.py | 10 ++++++++++ 6 files changed, 36 insertions(+), 2 deletions(-) diff --git a/httprunner/exceptions.py b/httprunner/exceptions.py index 036137da..3db701ee 100644 --- a/httprunner/exceptions.py +++ b/httprunner/exceptions.py @@ -47,6 +47,9 @@ class FunctionNotFound(NotFoundError): class VariableNotFound(NotFoundError): pass +class EnvNotFound(NotFoundError): + pass + class ApiNotFound(NotFoundError): pass diff --git a/httprunner/parser.py b/httprunner/parser.py index ff68352a..d7fb2f01 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -415,7 +415,9 @@ def parse_string_functions(content, variables_mapping, functions_mapping): if func_name in ["parameterize", "P"]: from httprunner import loader - eval_value = loader.load_csv_file(*args, **kwargs) + eval_value = loader.load_csv_file(*args) + elif func_name in ["environ", "ENV"]: + eval_value = utils.get_os_environ(*args) else: func = get_mapping_function(func_name, functions_mapping) eval_value = func(*args, **kwargs) diff --git a/httprunner/utils.py b/httprunner/utils.py index d1ccc4a8..5fd35d8f 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -29,6 +29,25 @@ def unset_os_environ(variables_mapping): logger.log_debug("Unset OS environment variable: {}".format(variable)) +def get_os_environ(variable_name): + """ get value of environment variable. + + Args: + variable_name(str): variable name + + Returns: + value of environment variable. + + Raises: + exceptions.EnvNotFound: If environment variable not found. + + """ + try: + return os.environ[variable_name] + except KeyError: + raise exceptions.EnvNotFound(variable_name) + + def query_json(json_content, query, delimiter='.'): """ Do an xpath-like query with json_content. @param (dict/list/string) json_content diff --git a/tests/data/demo_testcase.yml b/tests/data/demo_testcase.yml index 595185ef..e0f15764 100644 --- a/tests/data/demo_testcase.yml +++ b/tests/data/demo_testcase.yml @@ -3,6 +3,7 @@ variables: - var_a: 0 - var_c: "${sum_two(1, 2)}" + - PROJECT_KEY: ${ENV(PROJECT_KEY)} parameters: - "var_a-var_b": - [11, 21] diff --git a/tests/test_loader.py b/tests/test_loader.py index 2c6dcc49..91c279ed 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -27,7 +27,6 @@ class TestFileLoader(unittest.TestCase): os.remove(yaml_tmp_file) - def test_load_json_file_file_format_error(self): json_tmp_file = "tests/data/tmp.json" # create empty file diff --git a/tests/test_parser.py b/tests/test_parser.py index 9dd7a6fd..b3a1ef17 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -432,8 +432,18 @@ class TestParser(unittest.TestCase): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase.yml') testcases = loader.load_tests(testcase_file_path) + self.assertEqual( + testcases[0]["config"]["variables"][1]["var_c"], + "${sum_two(1, 2)}" + ) + self.assertEqual( + testcases[0]["config"]["variables"][2]["PROJECT_KEY"], + "${ENV(PROJECT_KEY)}" + ) + parsed_testcases = parser.parse_tests(testcases) self.assertEqual(parsed_testcases[0]["config"]["variables"]["var_c"], 3) + self.assertEqual(parsed_testcases[0]["config"]["variables"]["PROJECT_KEY"], "ABCDEFGH") self.assertEqual(len(parsed_testcases), 2 * 2) self.assertEqual( parsed_testcases[0]["config"]["request"]["base_url"], From 05eb07935a55eaa820ed56cb693b6117534a4b2a Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 16 Nov 2018 12:45:06 +0800 Subject: [PATCH 021/113] add args and kwargs validation for ENV() and P() --- httprunner/parser.py | 8 ++++++-- tests/test_parser.py | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/httprunner/parser.py b/httprunner/parser.py index d7fb2f01..50074bb7 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -414,10 +414,14 @@ def parse_string_functions(content, variables_mapping, functions_mapping): kwargs = parse_data(kwargs, variables_mapping, functions_mapping) if func_name in ["parameterize", "P"]: + if len(args) != 1 or kwargs: + raise exceptions.ParamsError("P() should only pass in one argument!") from httprunner import loader - eval_value = loader.load_csv_file(*args) + eval_value = loader.load_csv_file(args[0]) elif func_name in ["environ", "ENV"]: - eval_value = utils.get_os_environ(*args) + if len(args) != 1 or kwargs: + raise exceptions.ParamsError("ENV() should only pass in one argument!") + eval_value = utils.get_os_environ(args[0]) else: func = get_mapping_function(func_name, functions_mapping) eval_value = func(*args, **kwargs) diff --git a/tests/test_parser.py b/tests/test_parser.py index b3a1ef17..84d9e948 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -451,3 +451,28 @@ class TestParser(unittest.TestCase): ) self.assertIsInstance(parsed_testcases, list) self.assertEqual(parsed_testcases[0]["config"]["name"], '12311') + + def test_parse_environ(self): + os.environ["PROJECT_KEY"] = "ABCDEFGH" + content = { + "variables": [ + {"PROJECT_KEY": "${ENV(PROJECT_KEY)}"} + ] + } + result = parser.parse_data(content) + + content = { + "variables": [ + {"PROJECT_KEY": "${ENV(PROJECT_KEY, abc)}"} + ] + } + with self.assertRaises(exceptions.ParamsError): + parser.parse_data(content) + + content = { + "variables": [ + {"PROJECT_KEY": "${ENV(abc=123)}"} + ] + } + with self.assertRaises(exceptions.ParamsError): + parser.parse_data(content) From 82b527d8b2aa35ef8ef956c6697e2852d99a7896 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 16 Nov 2018 14:50:14 +0800 Subject: [PATCH 022/113] add unittest: variables in testcase teststep should override api's --- tests/api/basic.yml | 4 ++++ tests/test_loader.py | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/tests/api/basic.yml b/tests/api/basic.yml index 41ea3327..5009a1c8 100644 --- a/tests/api/basic.yml +++ b/tests/api/basic.yml @@ -17,6 +17,10 @@ - api: def: create_user($uid, $user_name, $user_password, $token) + variables: + - user_name: user0 + - user_password: "000000" + - uid: 2000 request: url: /api/users/$uid method: POST diff --git a/tests/test_loader.py b/tests/test_loader.py index 91c279ed..c0ff483b 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -465,6 +465,17 @@ class TestSuiteLoader(unittest.TestCase): testcases_list = loader.load_tests(path) self.assertIn("variables", testcases_list[0]["config"]) self.assertIn("request", testcases_list[0]["config"]) + + # variables in testcase teststep should override api's + self.assertEqual( + testcases_list[0]["teststeps"][3]["variables"][0]["user_name"], + "user1" + ) + self.assertEqual( + testcases_list[0]["teststeps"][3]["variables"][2]["uid"], + 2000 + ) + self.assertIn("request", testcases_list[0]["teststeps"][0]) self.assertIn("url", testcases_list[0]["teststeps"][0]["request"]) self.assertIn("validate", testcases_list[0]["teststeps"][0]) From 4099ade49d6775d11e41f7bf5eae1913c71bf832 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 22 Nov 2018 19:20:30 +0800 Subject: [PATCH 023/113] HttpRunner 2.0 is comming! 1, new design for testcase format; 2, refactor testcase layer mechanism. --- httprunner/__about__.py | 2 +- httprunner/api.py | 74 +-- httprunner/cli.py | 18 +- httprunner/context.py | 141 ++--- httprunner/loader.py | 567 ++++-------------- httprunner/parser.py | 412 ++++++++++--- httprunner/runner.py | 215 ++++--- httprunner/utils.py | 107 ++++ httprunner/validator.py | 47 +- tests/api/basic.yml | 59 +- tests/data/bugfix_parameters.yml | 2 +- tests/data/demo_parameters.yml | 10 +- tests/data/demo_testcase.yml | 11 +- tests/data/demo_testcase_functions.yml | 10 +- tests/data/demo_testcase_layer.yml | 60 +- tests/data/demo_testcase_variables.yml | 10 +- tests/httpbin/basic.yml | 3 +- tests/httpbin/hooks.yml | 3 +- tests/httpbin/load_image.yml | 3 +- tests/httpbin/upload.yml | 3 +- tests/test_api.py | 309 ++++++++-- tests/test_context.py | 87 +-- tests/test_loader.py | 358 +++++------ tests/test_parser.py | 56 +- tests/test_runner.py | 48 +- tests/test_utils.py | 66 +- tests/test_validator.py | 32 +- .../create_and_check.yml} | 30 +- tests/{suite => testcases}/setup.yml | 19 +- tests/testcases/smoketest.yml | 24 - tests/testsuites/create_users.yml | 17 + 31 files changed, 1500 insertions(+), 1303 deletions(-) rename tests/{suite/create_and_get.yml => testcases/create_and_check.yml} (54%) rename tests/{suite => testcases}/setup.yml (57%) delete mode 100644 tests/testcases/smoketest.yml create mode 100644 tests/testsuites/create_users.yml diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 50405f44..337d8cdd 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.6.0.alpha' +__version__ = '2.0.0.beta' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/api.py b/httprunner/api.py index b506d9f1..97ebde3b 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -28,17 +28,17 @@ class HttpRunner(object): self.test_loader = unittest.TestLoader() self.summary = None - def _add_tests(self, testcases): + def _add_tests(self, tests_mapping): """ initialize testcase with Runner() and add to test suite. Args: - testcases (list): parsed testcases list + tests_mapping (dict): project info and testcases list. Returns: unittest.TestSuite() """ - def _add_teststep(test_runner, config, teststep_dict): + def _add_teststep(test_runner, teststep_dict): """ add teststep to testcase. """ def test(self): @@ -52,22 +52,16 @@ class HttpRunner(object): self.meta_data["validators"] = test_runner.evaluated_validators test_runner.http_client_session.init_meta_data() - try: - teststep_dict["name"] = parser.parse_data( - teststep_dict["name"], - config.get("variables", {}), - config.get("functions", {}) - ) - except exceptions.VariableNotFound: - pass - - test.__doc__ = teststep_dict["name"] + # TODO: refactor + test.__doc__ = teststep_dict.get("name") or teststep_dict.get("config", {}).get("name") return test test_suite = unittest.TestSuite() - for testcase in testcases: + functions = tests_mapping.get("project_mapping", {}).get("functions", {}) + + for testcase in tests_mapping["testcases"]: config = testcase.get("config", {}) - test_runner = runner.Runner(config, self.http_client_session) + test_runner = runner.Runner(config, functions, self.http_client_session) TestSequense = type('TestSequense', (unittest.TestCase,), {}) teststeps = testcase.get("teststeps", []) @@ -76,12 +70,12 @@ class HttpRunner(object): # suppose one testcase should not have more than 9999 steps, # and one step should not run more than 999 times. test_method_name = 'test_{:04}_{:03}'.format(index, times_index) - test_method = _add_teststep(test_runner, config, teststep_dict) + test_method = _add_teststep(test_runner, teststep_dict) setattr(TestSequense, test_method_name, test_method) loaded_testcase = self.test_loader.loadTestsFromTestCase(TestSequense) setattr(loaded_testcase, "config", config) - setattr(loaded_testcase, "teststeps", testcase.get("teststeps", [])) + setattr(loaded_testcase, "teststeps", teststeps) setattr(loaded_testcase, "runner", test_runner) test_suite.addTest(loaded_testcase) @@ -140,49 +134,21 @@ class HttpRunner(object): self.summary["details"].append(testcase_summary) - def _run_tests(self, testcases, mapping=None): + def _run_tests(self, tests_mapping): """ start to run test with variables mapping. Args: - testcases (list): list of testcase_dict, each testcase is corresponding to a YAML/JSON file - [ - { # testcase data structure - "config": { - "name": "desc1", - "path": "testcase1_path", - "variables": [], # optional - "request": {}, # optional - "functions": {}, - "env": {}, - "def-api": {}, - "def-testcase": {} - }, - "teststeps": [ - # teststep data structure - { - 'name': 'test step desc2', - 'variables': [], # optional - 'extract': [], # optional - 'validate': [], - 'request': {}, - 'function_meta': {} - }, - teststep2 # another teststep dict - ] - }, - testcase_dict_2 # another testcase dict - ] - mapping (dict): if mapping is specified, it will override variables in config block. + tests_mapping (dict): list of testcase_dict, each testcase is corresponding to a YAML/JSON file Returns: instance: HttpRunner() instance """ self.exception_stage = "parse tests" - parsed_testcases_list = parser.parse_tests(testcases, mapping) + parser.parse_tests(tests_mapping) self.exception_stage = "add tests to test suite" - test_suite = self._add_tests(parsed_testcases_list) + test_suite = self._add_tests(tests_mapping) self.exception_stage = "run test suite" results = self._run_suite(test_suite) @@ -207,16 +173,14 @@ class HttpRunner(object): self.exception_stage = "load tests" if validator.is_testcases(path_or_testcases): - if isinstance(path_or_testcases, dict): - testcases = [path_or_testcases] - else: - testcases = path_or_testcases + tests_mapping = path_or_testcases elif validator.is_testcase_path(path_or_testcases): - testcases = loader.load_tests(path_or_testcases, dot_env_path) + tests_mapping = loader.load_tests(path_or_testcases, dot_env_path) + tests_mapping["project_mapping"]["variables"] = mapping or {} else: raise exceptions.ParamsError("invalid testcase path or testcases.") - return self._run_tests(testcases, mapping) + return self._run_tests(tests_mapping) def gen_html_report(self, html_report_name=None, html_report_template=None): """ generate html report and return report path. diff --git a/httprunner/cli.py b/httprunner/cli.py index 079eff17..d242d456 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -77,17 +77,13 @@ def main_hrun(): create_scaffold(project_name) exit(0) - try: - runner = HttpRunner( - failfast=args.failfast - ) - runner.run( - args.testcase_paths, - dot_env_path=args.dot_env_path - ) - except Exception: - logger.log_error("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage)) - raise + for path in args.testcase_paths: + try: + runner = HttpRunner(failfast=args.failfast) + runner.run(path, dot_env_path=args.dot_env_path) + except Exception: + logger.log_error("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage)) + raise if not args.no_html_report: runner.gen_html_report( diff --git a/httprunner/context.py b/httprunner/context.py index 45dd3b51..c4d98546 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -1,75 +1,59 @@ -# encoding: utf-8 - -import copy - from httprunner import exceptions, logger, parser, utils -class Context(object): - """ Manages context functions and variables. - context has two levels, testcase and teststep. +class SessionContext(object): + """ HttpRunner session, store runtime variables. + + Examples: + >>> functions={...} + >>> variables = {"SECRET_KEY": "DebugTalk"} + >>> context = SessionContext(functions, variables) + + Equivalent to: + >>> context = SessionContext(functions) + >>> context.update_seesion_variables(variables) + """ - def __init__(self, variables=None, functions=None): - """ init Context with testcase variables and functions. - """ - variables = variables or {} - functions = functions or {} - # testcase level context - ## TESTCASE_SHARED_VARIABLES_MAPPING and TESTCASE_SHARED_FUNCTIONS_MAPPING are unchangeable. - self.TESTCASE_SHARED_VARIABLES_MAPPING = utils.ensure_mapping_format(variables) - self.TESTCASE_SHARED_FUNCTIONS_MAPPING = functions + def __init__(self, functions, variables=None): + self.session_variables_mapping = utils.ensure_mapping_format(variables or {}) + self.FUNCTIONS_MAPPING = functions + self.teststep_variables_mapping = {} + self.init_teststep_variables() - # testcase level request, will not change - self.TESTCASE_SHARED_REQUEST_MAPPING = {} - - self.evaluated_validators = [] - self.init_context_variables(level="testcase") - - def init_context_variables(self, level="testcase"): - """ initialize testcase/teststep context + def init_teststep_variables(self, variables_mapping=None): + """ init teststep variables, called when each teststep(api) starts. + variables_mapping will be evaluated first. Args: - level (enum): "testcase" or "teststep" - - """ - if level == "testcase": - # testcase level runtime context, will be updated with extracted variables in each teststep. - self.testcase_runtime_variables_mapping = copy.deepcopy(self.TESTCASE_SHARED_VARIABLES_MAPPING) - - # teststep level context, will be altered in each teststep. - # teststep config shall inherit from testcase configs, - # but can not change testcase configs, that's why we use copy.deepcopy here. - self.teststep_variables_mapping = copy.deepcopy(self.testcase_runtime_variables_mapping) - - def update_context_variables(self, variables, level): - """ update context variables, with level specified. - - Args: - variables (list/OrderedDict): testcase config block or teststep block + variables_mapping (dict/list) [ {"TOKEN": "debugtalk"}, {"random": "${gen_random_string(5)}"}, - {"json": {'name': 'user', 'password': '123456'}}, - {"md5": "${gen_md5($TOKEN, $json, $random)}"} + {"data": '{"name": "user", "password": "123456"}'}, + {"authorization": "${gen_md5($TOKEN, $data, $random)}"} ] - OrderDict({ - "TOKEN": "debugtalk", - "random": "${gen_random_string(5)}", - "json": {'name': 'user', 'password': '123456'}, - "md5": "${gen_md5($TOKEN, $json, $random)}" - }) - level (enum): "testcase" or "teststep" """ - variables_mapping = utils.ensure_mapping_format(variables) - + variables_mapping = variables_mapping or {} + variables_mapping = utils.ensure_mapping_format(variables_mapping) for variable_name, variable_value in variables_mapping.items(): - variable_eval_value = self.eval_content(variable_value) + variable_value = self.eval_content(variable_value) + self.update_teststep_variables(variable_name, variable_value) - if level == "testcase": - self.testcase_runtime_variables_mapping[variable_name] = variable_eval_value + self.teststep_variables_mapping.update(self.session_variables_mapping) - self.update_teststep_variables_mapping(variable_name, variable_eval_value) + def update_teststep_variables(self, variable_name, variable_value): + """ update teststep variables, these variables are only valid in the current teststep. + """ + self.teststep_variables_mapping[variable_name] = variable_value + + def update_seesion_variables(self, variables_mapping): + """ update session with extracted variables mapping. + these variables are valid in the whole running session. + """ + variables_mapping = utils.ensure_mapping_format(variables_mapping) + self.session_variables_mapping.update(variables_mapping) + self.teststep_variables_mapping.update(self.session_variables_mapping) def eval_content(self, content): """ evaluate content recursively, take effect on each variable and function in content. @@ -78,50 +62,9 @@ class Context(object): return parser.parse_data( content, self.teststep_variables_mapping, - self.TESTCASE_SHARED_FUNCTIONS_MAPPING + self.FUNCTIONS_MAPPING ) - def update_testcase_runtime_variables_mapping(self, variables): - """ update testcase_runtime_variables_mapping with extracted vairables in teststep. - - Args: - variables (OrderDict): extracted variables in teststep - - """ - for variable_name, variable_value in variables.items(): - self.testcase_runtime_variables_mapping[variable_name] = variable_value - self.update_teststep_variables_mapping(variable_name, variable_value) - - def update_teststep_variables_mapping(self, variable_name, variable_value): - """ bind and update testcase variables mapping - """ - self.teststep_variables_mapping[variable_name] = variable_value - - def get_parsed_request(self, request_dict, level="teststep"): - """ get parsed request with variables and functions. - - Args: - request_dict (dict): request config mapping - level (enum): "testcase" or "teststep" - - Returns: - dict: parsed request dict - - """ - if level == "testcase": - # testcase config request dict has been parsed in parse_tests - self.TESTCASE_SHARED_REQUEST_MAPPING = copy.deepcopy(request_dict) - return self.TESTCASE_SHARED_REQUEST_MAPPING - - else: - # teststep - return self.eval_content( - utils.deep_update_dict( - copy.deepcopy(self.TESTCASE_SHARED_REQUEST_MAPPING), - request_dict - ) - ) - def __eval_check_item(self, validator, resp_obj): """ evaluate check item in validator. @@ -183,7 +126,7 @@ class Context(object): """ # TODO: move comparator uniform to init_test_suites comparator = utils.get_uniform_comparator(validator_dict["comparator"]) - validate_func = parser.get_mapping_function(comparator, self.TESTCASE_SHARED_FUNCTIONS_MAPPING) + validate_func = parser.get_mapping_function(comparator, self.FUNCTIONS_MAPPING) check_item = validator_dict["check"] check_value = validator_dict["check_value"] diff --git a/httprunner/loader.py b/httprunner/loader.py index a1535640..afc79aa0 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -273,15 +273,21 @@ def load_debugtalk_functions(): ## testcase loader ############################################################################### -def _load_teststeps(test_block, project_mapping): - """ load teststeps with api/testcase references +project_mapping = {} +tests_def_mapping = { + "api": {}, + "testcases": {} +} + +def load_teststep(raw_stepinfo): + """ load teststep with api/testcase/proc references Args: - test_block (dict): test block content, maybe in 3 formats. + raw_stepinfo (dict): teststep data, maybe in 3 formats. # api reference { "name": "add product to cart", - "api": "api_add_cart()", + "api": "api_add_cart", "variables": [], "validate": [], "extract": [] @@ -289,7 +295,7 @@ def _load_teststeps(test_block, project_mapping): # testcase reference { "name": "add product to cart", - "suite": "create_and_check()", + "testcase": "create_and_check", "variables": [] } # define directly @@ -304,33 +310,43 @@ def _load_teststeps(test_block, project_mapping): Returns: list: loaded teststeps list - """ - teststeps = [] + Args: + raw_stepinfo (dict): teststep info + """ # reference api - if "api" in test_block: - ref_call = test_block.pop("api") - def_block = _get_block_by_name(ref_call, "def-api", project_mapping) - extended_block = _extend_block(test_block, def_block) - teststeps.append(extended_block) + if "api" in raw_stepinfo: + api_name = raw_stepinfo["api"] + raw_stepinfo["api_def"] = _get_api_definition(api_name) + + # TODO: reference proc functions + elif "func" in raw_stepinfo: + pass # reference testcase - elif "suite" in test_block: # TODO: replace suite with testcase - ref_call = test_block.pop("suite") - def_block = _get_block_by_name(ref_call, "def-testcase", project_mapping) - # TODO: bugfix lost block config variables - for teststep in def_block["teststeps"]: - _teststeps = _load_teststeps(teststep, project_mapping) - teststeps.extend(_teststeps) + elif "testcase" in raw_stepinfo: + testcase_path = raw_stepinfo["testcase"] + + if testcase_path not in tests_def_mapping["testcases"]: + testcase_path = os.path.join( + project_mapping["PWD"], + testcase_path + ) + testcase_dict = load_testcase(load_file(testcase_path)) + tests_def_mapping[testcase_path] = testcase_dict + else: + testcase_dict = tests_def_mapping[testcase_path] + + raw_stepinfo["testcase_def"] = testcase_dict # define directly else: - teststeps.append(test_block) + pass - return teststeps + return raw_stepinfo -def _load_testcase(raw_testcase, project_mapping): +def load_testcase(raw_testcase): """ load testcase/testsuite with api/testcase references Args: @@ -352,20 +368,18 @@ def _load_testcase(raw_testcase, project_mapping): "test": {...} } ] - project_mapping (dict): project_mapping Returns: dict: loaded testcase content { + "name": "XYZ", "config": {}, "teststeps": [teststep11, teststep12] } """ - loaded_testcase = { - "config": {}, - "teststeps": [] - } + config = {} + teststeps = [] for item in raw_testcase: # TODO: add json schema validation @@ -377,290 +391,38 @@ def _load_testcase(raw_testcase, project_mapping): raise exceptions.FileFormatError("Testcase format error: {}".format(item)) if key == "config": - loaded_testcase["config"].update(test_block) + config.update(test_block) elif key == "test": - loaded_testcase["teststeps"].extend(_load_teststeps(test_block, project_mapping)) + teststeps.append(load_teststep(test_block)) else: logger.log_warning( "unexpected block key: {}. block key should only be 'config' or 'test'.".format(key) ) - return loaded_testcase + return { + "config": config, + "teststeps": teststeps + } -def _get_block_by_name(ref_call, ref_type, project_mapping): - """ get test content by reference name. - - Args: - ref_call (str): call function. - e.g. api_v1_Account_Login_POST($UserName, $Password) - ref_type (enum): "def-api" or "def-testcase" - project_mapping (dict): project_mapping +def _get_api_definition(name): + """ get api definition by name. Returns: - dict: api/testcase definition. - - Raises: - exceptions.ParamsError: call args number is not equal to defined args number. - - """ - function_meta = parser.parse_function(ref_call) - func_name = function_meta["func_name"] - call_args = function_meta["args"] - block = _get_test_definition(func_name, ref_type, project_mapping) - def_args = block.get("function_meta", {}).get("args", []) - - if len(call_args) != len(def_args): - err_msg = "{}: call args number is not equal to defined args number!\n".format(func_name) - err_msg += "defined args: {}\n".format(def_args) - err_msg += "reference args: {}".format(call_args) - logger.log_error(err_msg) - raise exceptions.ParamsError(err_msg) - - args_mapping = {} - for index, item in enumerate(def_args): - if call_args[index] == item: - continue - - args_mapping[item] = call_args[index] - - if args_mapping: - block = parser.substitute_variables(block, args_mapping) - - return block - - -def _get_test_definition(name, ref_type, project_mapping): - """ get expected api or testcase. - - Args: - name (str): api or testcase name - ref_type (enum): "def-api" or "def-testcase" - project_mapping (dict): project_mapping - - Returns: - dict: expected api/testcase info if found. + dict: expected api definition if found. Raises: exceptions.ApiNotFound: api not found - exceptions.TestcaseNotFound: testcase not found """ - block = project_mapping.get(ref_type, {}).get(name) - # NOTICE: avoid project_mapping been changed during iteration. - block = copy.deepcopy(block) - - if not block: - err_msg = "{} not found!".format(name) - if ref_type == "def-api": - raise exceptions.ApiNotFound(err_msg) - else: - # ref_type == "def-testcase": - raise exceptions.TestcaseNotFound(err_msg) - - return block - - -def _extend_block(ref_block, def_block): - """ extend ref_block with def_block, ref_block will merge and override def_block. - - Args: - def_block (dict): api definition dict. - ref_block (dict): reference block - - Returns: - dict: extended reference block. - - Examples: - >>> def_block = { - "name": "get token 1", - "request": {...}, - "validate": [{'eq': ['status_code', 200]}] - } - >>> ref_block = { - "name": "get token 2", - "extract": [{"token": "content.token"}], - "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] - } - >>> _extend_block(def_block, ref_block) - { - "name": "get token 2", - "request": {...}, - "extract": [{"token": "content.token"}], - "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] - } - - """ - extended_block = {} - - # override name - extended_block["name"] = ref_block.pop("name", None) or def_block.pop("name", "") - - # override variables - def_variables = def_block.pop("variables", []) - ref_variables = ref_block.pop("variables", []) - extended_block["variables"] = _extend_variables( - def_variables, - ref_variables - ) - - # merge & override validators - def_validators = def_block.pop("validate", []) - ref_validators = ref_block.pop("validate", []) - extended_block["validate"] = _extend_validators( - def_validators, - ref_validators - ) - - # merge & override extractors - def_extrators = def_block.pop("extract", []) - ref_extractors = ref_block.pop("extract", []) - extended_block["extract"] = _extend_variables( - def_extrators, - ref_extractors - ) - - # TODO: merge & override request - def_request = def_block.pop("request", {}) - ref_request = ref_block.pop("request", {}) - extended_block["request"] = def_request - - # merge & override setup_hooks - def_setup_hooks = def_block.pop("setup_hooks", []) - ref_setup_hooks = ref_block.pop("setup_hooks", []) - extended_block["setup_hooks"] = list(set(def_setup_hooks + ref_setup_hooks)) - # merge & override teardown_hooks - def_teardown_hooks = def_block.pop("teardown_hooks", []) - ref_teardown_hooks = ref_block.pop("teardown_hooks", []) - extended_block["teardown_hooks"] = list(set(def_teardown_hooks + ref_teardown_hooks)) - - # TODO: extend with other ref block items, e.g. times - # extended_block.update(def_block) - extended_block.update(ref_block) - - return extended_block - - -def _convert_validators_to_mapping(validators): - """ convert validators list to mapping. - - Args: - validators (list): validators in list - - Returns: - dict: validators mapping, use (check, comparator) as key. - - Examples: - >>> validators = [ - {"check": "v1", "expect": 201, "comparator": "eq"}, - {"check": {"b": 1}, "expect": 200, "comparator": "eq"} - ] - >>> _convert_validators_to_mapping(validators) - { - ("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"}, - ('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"} - } - - """ - validators_mapping = {} - - for validator in validators: - validator = parser.parse_validator(validator) - - if not isinstance(validator["check"], collections.Hashable): - check = json.dumps(validator["check"]) - else: - check = validator["check"] - - key = (check, validator["comparator"]) - validators_mapping[key] = validator - - return validators_mapping - - -def _extend_validators(def_validators, ref_validators): - """ extend ref_validators with def_validators. - ref_validators will merge and override def_validators. - - Args: - def_validators (list): - ref_validators (list): - - Returns: - list: extended validators - - Examples: - >>> def_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] - >>> ref_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] - >>> _extend_validators(def_validators, ref_validators) - [ - {"check": "v1", "expect": 201, "comparator": "eq"}, - {"check": "s2", "expect": 16, "comparator": "len_eq"}, - {"check": "s3", "expect": 12, "comparator": "len_eq"} - ] - - """ - if not def_validators: - return ref_validators - - elif not ref_validators: - return def_validators - - else: - def_validators_mapping = _convert_validators_to_mapping(def_validators) - ref_validators_mapping = _convert_validators_to_mapping(ref_validators) - - def_validators_mapping.update(ref_validators_mapping) - return list(def_validators_mapping.values()) - - -def _extend_variables(def_variables, ref_variables): - """ extend ref_variables with def_variables. - ref_variables will merge and override def_variables. - - Args: - def_variables (list): - ref_variables (list): - - Returns: - list: extended variables - - Examples: - >>> def_variables = [{"var1": "val1"}, {"var2": "val2"}] - >>> ref_variables = [{"var1": "val111"}, {"var3": "val3"}] - >>> _extend_variables(def_variables, ref_variables) - [ - {"var1": "val111"}, - {"var2": "val2"}, - {"var3": "val3"} - ] - - """ - if not def_variables: - return ref_variables - - elif not ref_variables: - return def_variables - - else: - extended_variables_dict = OrderedDict() - for def_variable in def_variables: - var_name = list(def_variable.keys())[0] - extended_variables_dict[var_name] = def_variable[var_name] - - for ref_variable in ref_variables: - if not ref_variable: - continue - var_name = list(ref_variable.keys())[0] - extended_variables_dict[var_name] = ref_variable[var_name] - - extended_variables = [] - for key, value in extended_variables_dict.items(): - extended_variables.append({key: value}) - - return extended_variables + try: + block = tests_def_mapping["api"][name] + # NOTICE: avoid project_mapping been changed during iteration. + return utils.deepcopy_dict(block) + except KeyError: + raise exceptions.ApiNotFound("{} not found!".format(name)) def load_folder_content(folder_path): @@ -737,99 +499,16 @@ def load_api_folder(api_folder_path): for api_item in api_items: key, api_dict = api_item.popitem() - # TODO: replace def with api file path - api_def = api_dict.pop("def") - function_meta = parser.parse_function(api_def) - func_name = function_meta["func_name"] + # TODO: replace id with api file path + api_id = api_dict.get("id") + if api_id in api_definition_mapping: + logger.log_warning("API definition duplicated: {}".format(api_id)) - if func_name in api_definition_mapping: - logger.log_warning("API definition duplicated: {}".format(func_name)) - - api_dict["function_meta"] = function_meta - api_definition_mapping[func_name] = api_dict + api_definition_mapping[api_id] = api_dict return api_definition_mapping -def load_test_folder(test_folder_path): - """ load testcases definitions from folder. - - Args: - test_folder_path (str): testcases files folder. - - testcase file should be in the following format: - [ - { - "config": { - "def": "create_and_check", - "request": {}, - "validate": [] - } - }, - { - "test": { - "api": "get_user", - "validate": [] - } - } - ] - - Returns: - dict: testcases definition mapping. - - { - "create_and_check": [ - {"config": {}}, - {"test": {}}, - {"test": {}} - ], - "tests/testcases/create_and_get.yml": [ - {"config": {}}, - {"test": {}}, - {"test": {}} - ] - } - - """ - test_definition_mapping = {} - - test_items_mapping = load_folder_content(test_folder_path) - - for test_file_path, items in test_items_mapping.items(): - # TODO: add JSON schema validation - - testcase = { - "config": {}, - "teststeps": [] - } - for item in items: - key, block = item.popitem() - - if key == "config": - testcase["config"].update(block) - - if "def" not in block: - test_definition_mapping[test_file_path] = testcase - continue - - # TODO: replace def with testcase file path - testcase_def = block.pop("def") - function_meta = parser.parse_function(testcase_def) - func_name = function_meta["func_name"] - - if func_name in test_definition_mapping: - logger.log_warning("API definition duplicated: {}".format(func_name)) - - testcase["function_meta"] = function_meta - test_definition_mapping[func_name] = testcase - else: - # key == "test": - ### TODO: extend suite with api - testcase["teststeps"].append(block) - - return test_definition_mapping - - def load_debugtalk_py(start_path): """ locate debugtalk.py file and returns PWD and debugtalk.py functions. @@ -878,22 +557,17 @@ def load_project_tests(test_path, dot_env_path=None): dict: project loaded api/testcases definitions, environments and debugtalk.py functions. """ - project_mapping = {} - # locate PWD and load debugtalk.py functions project_working_directory, debugtalk_functions = load_debugtalk_py(test_path) + project_mapping["PWD"] = project_working_directory project_mapping["functions"] = debugtalk_functions # load .env dot_env_path = dot_env_path or os.path.join(project_working_directory, ".env") project_mapping["env"] = load_dot_env_file(dot_env_path) - # load api/testcases - project_mapping["def-api"] = load_api_folder(os.path.join(project_working_directory, "api")) - # TODO: replace suite with testcases - project_mapping["def-testcase"] = load_test_folder(os.path.join(project_working_directory, "suite")) - - return project_mapping + # load api + tests_def_mapping["api"] = load_api_folder(os.path.join(project_working_directory, "api")) def load_tests(path, dot_env_path=None): @@ -901,54 +575,44 @@ def load_tests(path, dot_env_path=None): Args: path (str/list): testcase file/foler path. - path could be in several types: + path could be in 2 types: - absolute/relative file path - absolute/relative folder path - - list/set container with file(s) and/or folder(s) dot_env_path (str): specified .env file path Returns: - list: testcases list, each testcase is corresponding to a file - [ - { # testcase data structure - "config": { - "name": "desc1", - "path": "testcase1_path", - "variables": [], # optional - "request": {}, # optional + dict: tests mapping, include project_mapping and testcases. + each testcase is corresponding to a file. + { + "project_mapping": { + "PWD": "XXXXX", "functions": {}, - "env": {}, - "def-api": {}, - "def-testcase": {} + "env": {} }, - "teststeps": [ - # teststep data structure - { - 'name': 'test step desc1', - 'variables': [], # optional - 'extract': [], # optional - 'validate': [], - 'request': {}, - 'function_meta': {} + "testcases": [ + { # testcase data structure + "config": { + "name": "desc1", + "path": "testcase1_path", + "variables": [], # optional + }, + "teststeps": [ + # teststep data structure + { + 'name': 'test step desc1', + 'variables': [], # optional + 'extract': [], # optional + 'validate': [], + 'request': {} + }, + teststep2 # another teststep dict + ] }, - teststep2 # another teststep dict + testcase_dict_2 # another testcase dict ] - }, - testcase_dict_2 # another testcase dict - ] + } """ - if isinstance(path, (list, set)): - testcases_list = [] - - for file_path in set(path): - testcases = load_tests(file_path, dot_env_path) - if not testcases: - continue - testcases_list.extend(testcases) - - return testcases_list - if not os.path.exists(path): err_msg = "path not exist: {}".format(path) logger.log_error(err_msg) @@ -957,22 +621,41 @@ def load_tests(path, dot_env_path=None): if not os.path.isabs(path): path = os.path.join(os.getcwd(), path) + load_project_tests(path, dot_env_path) + tests_mapping = { + "project_mapping": project_mapping + } + + def load_test_file(path): + raw_testcase = load_file(path) + + try: + testcase = load_testcase(raw_testcase) + testcase["config"]["path"] = path + except exceptions.FileFormatError: + testcase = {} + + return testcase + + testcases_list = [] + if os.path.isdir(path): files_list = load_folder_files(path) - testcases_list = load_tests(files_list, dot_env_path) + for path in files_list: + testcase = load_test_file(path) + if not testcase: + continue + testcases_list.append(testcase) elif os.path.isfile(path): - try: - raw_testcase = load_file(path) - project_mapping = load_project_tests(path, dot_env_path) - testcase = _load_testcase(raw_testcase, project_mapping) - testcase["config"]["path"] = path - testcase["config"].update(project_mapping) - testcases_list = [testcase] - except exceptions.FileFormatError: - testcases_list = [] - return testcases_list + testcase = load_test_file(path) + if testcase: + testcases_list.append(testcase) + + tests_mapping["testcases"] = testcases_list + + return tests_mapping def load_locust_tests(path, dot_env_path=None): @@ -999,7 +682,7 @@ def load_locust_tests(path, dot_env_path=None): """ raw_testcase = load_file(path) - project_mapping = load_project_tests(path, dot_env_path) + load_project_tests(path, dot_env_path) config = {} tests = [] @@ -1009,10 +692,10 @@ def load_locust_tests(path, dot_env_path=None): if key == "config": config.update(test_block) elif key == "test": - teststeps = _load_teststeps(test_block, project_mapping) - weight = test_block.pop("weight", 1) + teststep = load_teststep(test_block) + weight = test_block.get("weight", 1) for _ in range(weight): - tests.append(teststeps) + tests.append(teststep) # parse config variables raw_config_variables = config.get("variables", []) diff --git a/httprunner/parser.py b/httprunner/parser.py index 50074bb7..63aac7d8 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -3,6 +3,7 @@ import ast import os import re +from collections import OrderedDict from httprunner import exceptions, utils from httprunner.compat import basestring, builtin_str, numeric_types, str @@ -516,7 +517,7 @@ def parse_data(content, variables_mapping=None, functions_mapping=None): for item in content ] - if isinstance(content, dict): + if isinstance(content, (dict, OrderedDict)): parsed_content = {} for key, value in content.items(): parsed_key = parse_data(key, variables_mapping, functions_mapping) @@ -527,7 +528,7 @@ def parse_data(content, variables_mapping=None, functions_mapping=None): if isinstance(content, basestring): # content is in string format here - variables_mapping = variables_mapping or {} + variables_mapping = utils.ensure_mapping_format(variables_mapping or {}) functions_mapping = functions_mapping or {} content = content.strip() @@ -541,110 +542,327 @@ def parse_data(content, variables_mapping=None, functions_mapping=None): return content -def parse_tests(testcases, variables_mapping=None): - """ parse testcases configs, including variables/parameters/name/request. +def _extend_with_api(teststep_dict, api_def_dict): + """ extend teststep with api definition, teststep will merge and override api definition. Args: - testcases (list): testcase list, with config unparsed. - [ - { # testcase data structure - "config": { - "name": "desc1", - "path": "testcase1_path", - "variables": {}, # optional - "request": {} # optional - "functions": {}, - "env": {}, - "def-api": {}, - "def-testcase": {} - }, - "teststeps": [ - # teststep data structure - { - 'name': 'test step desc2', - 'variables': [], # optional - 'extract': [], # optional - 'validate': [], - 'request': {}, - 'function_meta': {} - }, - teststep2 # another teststep dict - ] + teststep_dict (dict): teststep block + api_def_dict (dict): api definition + + Returns: + dict: extended teststep dict. + + Examples: + >>> api_def_dict = { + "name": "get token 1", + "request": {...}, + "validate": [{'eq': ['status_code', 200]}] + } + >>> teststep_dict = { + "name": "get token 2", + "extract": [{"token": "content.token"}], + "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] + } + >>> _extend_with_api(teststep_dict, api_def_dict) + { + "name": "get token 2", + "request": {...}, + "extract": [{"token": "content.token"}], + "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] + } + + """ + # override name + api_def_name = api_def_dict.pop("name", "") + teststep_dict["name"] = teststep_dict.get("name") or api_def_name + + # override variables + def_variables = api_def_dict.pop("variables", []) + teststep_dict["variables"] = utils.extend_variables( + def_variables, + teststep_dict.get("variables", {}) + ) + + # merge & override validators TODO: relocate + def_raw_validators = api_def_dict.pop("validate", []) + ref_raw_validators = teststep_dict.get("validate", []) + def_validators = [ + parse_validator(validator) + for validator in def_raw_validators + ] + ref_validators = [ + parse_validator(validator) + for validator in ref_raw_validators + ] + teststep_dict["validate"] = utils.extend_validators( + def_validators, + ref_validators + ) + + # merge & override extractors + def_extrators = api_def_dict.pop("extract", []) + teststep_dict["extract"] = utils.extend_variables( + def_extrators, + teststep_dict.get("extract", []) + ) + + # TODO: merge & override request + teststep_dict["request"] = api_def_dict.pop("request", {}) + # base_url + base_url = teststep_dict.pop("base_url", None) + if base_url: + teststep_dict["request"]["url"] = "{}/{}".format(base_url.rstrip("/"), teststep_dict["request"]["url"].lstrip("/")) + + # verify + if "verify" in teststep_dict: + verify = teststep_dict.pop("verify") + elif "verify" in api_def_dict: + verify = api_def_dict.pop("verify") + else: + verify = True + teststep_dict["request"]["verify"] = verify + + # merge & override setup_hooks + def_setup_hooks = api_def_dict.pop("setup_hooks", []) + ref_setup_hooks = teststep_dict.get("setup_hooks", []) + extended_setup_hooks = list(set(def_setup_hooks + ref_setup_hooks)) + if extended_setup_hooks: + teststep_dict["setup_hooks"] = extended_setup_hooks + # merge & override teardown_hooks + def_teardown_hooks = api_def_dict.pop("teardown_hooks", []) + ref_teardown_hooks = teststep_dict.get("teardown_hooks", []) + extended_teardown_hooks = list(set(def_teardown_hooks + ref_teardown_hooks)) + if extended_teardown_hooks: + teststep_dict["teardown_hooks"] = extended_teardown_hooks + + # TODO: extend with other api definition items, e.g. times + teststep_dict.update(api_def_dict) + + return teststep_dict + + +def _extend_with_testcase(teststep_dict, testcase_def_dict): + """ extend teststep with testcase definition + teststep will merge and override testcase config definition. + + Args: + teststep_dict (dict): teststep block + testcase_def_dict (dict): testcase definition + + Returns: + dict: extended teststep dict. + + """ + # override testcase config variables + testcase_def_dict["config"].setdefault("variables", {}) + testcase_def_variables = utils.ensure_mapping_format(testcase_def_dict["config"].get("variables", {})) + testcase_def_variables.update(teststep_dict.pop("variables", {})) + testcase_def_dict["config"]["variables"] = testcase_def_variables + + # override base_url, verify + # priority: testcase config > testsuite teststep + teststep_base_url = teststep_dict.pop("base_url", None) + teststep_verify = teststep_dict.pop("verify", True) + testcase_def_dict["config"].setdefault("base_url", teststep_base_url) + testcase_def_dict["config"].setdefault("verify", teststep_verify) + + # override testcase config name, output, etc. + testcase_def_dict["config"].update(teststep_dict) + + teststep_dict.clear() + teststep_dict.update(testcase_def_dict) + + +def __parse_config(config, project_mapping): + """ parse testcase config, include variables and name. + """ + # get config variables + raw_config_variables = config.pop("variables", {}) + raw_config_variables_mapping = utils.ensure_mapping_format(raw_config_variables) + override_variables = utils.deepcopy_dict(project_mapping.get("variables", {})) + functions = project_mapping.get("functions", {}) + + # override testcase config variables with passed in variables + for key, value in raw_config_variables_mapping.items(): + + if key in override_variables: + # passed in + continue + else: + # config variables + try: + parsed_value = parse_data( + value, + override_variables, + functions + ) + except exceptions.VariableNotFound: + pass + override_variables[key] = parsed_value + + if override_variables: + config["variables"] = override_variables + + # parse config name + config["name"] = parse_data( + config.get("name", ""), + override_variables, + functions + ) + + # parse config base_url + if "base_url" in config: + config["base_url"] = parse_data( + config["base_url"], + override_variables, + functions + ) + + +def __parse_teststeps(teststeps, config, project_mapping): + """ override teststeps with testcase config variables, base_url and verify. + teststep maybe nested testcase. + + variables priority: + testsuite config > testsuite teststep > testcase config > testcase teststep > api + + base_url/verify priority: + testcase teststep > testcase config > testsuite teststep > testsuite config > api + + Args: + teststeps (list): + config (dict): + + Returns: + list: overrided teststeps + + """ + config_variables = config.pop("variables", {}) + config_base_url = config.pop("base_url", None) + config_verify = config.pop("verify", True) + functions = project_mapping.get("functions", {}) + + for teststep in teststeps: + + # priority teststep > config + teststep.setdefault("base_url", config_base_url) + teststep.setdefault("verify", config_verify) + + if "testcase_def" in teststep: + # teststep is nested testcase + + # 1, testsuite config => testsuite teststeps + # override teststep variables + teststep["variables"] = utils.extend_variables( + teststep.pop("variables", {}), + config_variables + ) + + # parse teststep name + try: + teststep["name"] = parse_data( + teststep.pop("name", ""), + teststep["variables"], + functions + ) + except exceptions.VariableNotFound: + pass + + # 2, testsuite teststep => testcase config + testcase_def = teststep.pop("testcase_def") + _extend_with_testcase(teststep, testcase_def) + + # 3, testcase config => testcase teststep + _parse_testcase(teststep, project_mapping) + + else: + # 1, config => teststeps + # override teststep variables + teststep["variables"] = utils.extend_variables( + teststep.pop("variables", {}), + config_variables + ) + + # parse teststep name + try: + teststep["name"] = parse_data( + teststep.pop("name", ""), + teststep["variables"], + functions + ) + except exceptions.VariableNotFound: + pass + + if "api_def" in teststep: + # 2, teststep => api + api_def_dict = teststep.pop("api_def") + _extend_with_api(teststep, api_def_dict) + else: + # base_url + base_url = teststep.pop("base_url", None) + if base_url: + teststep["request"]["url"] = "{}/{}".format(base_url.rstrip("/"), teststep["request"]["url"].lstrip("/")) + + +def _parse_testcase(testcase, project_mapping): + __parse_config(testcase["config"], project_mapping) + __parse_teststeps(testcase["teststeps"], testcase["config"], project_mapping) + + +def parse_tests(tests_mapping): + """ parse testcases configs, including variables/name/request. + + Args: + tests_mapping (dict): project info and testcases list. + + { + "project_mapping": { + "PWD": "XXXXX", + "functions": {}, + "variables": {}, # optional, priority 1 + "env": {} }, - testcase_dict_2 # another testcase dict - ] + "testcases": [ + { # testcase data structure + "config": { + "name": "desc1", + "path": "testcase1_path", + "variables": [], # optional, priority 2 + }, + "teststeps": [ + # teststep data structure + { + 'name': 'test step desc1', + 'variables': [], # optional, priority 3 + 'extract': [], + 'validate': [], + 'api_def': { + "variables": {} # optional, priority 4 + 'request': {}, + } + }, + teststep2 # another teststep dict + ] + }, + testcase_dict_2 # another testcase dict + ] + } + variables_mapping (dict): if variables_mapping is specified, it will override variables in config block. Returns: list: parsed testcases list, with config variables/parameters/name/request parsed. """ - variables_mapping = variables_mapping or {} - parsed_testcases_list = [] + project_mapping = tests_mapping.get("project_mapping", {}) - for testcase in testcases: - testcase_config = testcase.setdefault("config", {}) - functions = testcase_config.get("functions", {}) + env_mapping = project_mapping.get("env", {}) + # set OS environment variables + utils.set_os_environ(env_mapping) - env_mapping = testcase_config.get("env", {}) - # set OS environment variables - utils.set_os_environ(env_mapping) + for testcase in tests_mapping["testcases"]: + testcase.setdefault("config", {}) + _parse_testcase(testcase, project_mapping) - # parse config parameters - config_parameters = testcase_config.pop("parameters", []) - cartesian_product_parameters_list = parse_parameters( - config_parameters, - testcase_config.get("variables", {}), - functions - ) or [{}] - - for parameter_mapping in cartesian_product_parameters_list: - testcase_dict = utils.deepcopy_dict(testcase) - config = testcase_dict.get("config") - # get config variables - raw_config_variables = config.get("variables", []) - raw_config_variables_mapping = utils.ensure_mapping_format(raw_config_variables) - - # priority: passed in > parameters > variables - - config_variables = utils.deepcopy_dict(parameter_mapping) - config_variables.update(variables_mapping) - - for key, value in raw_config_variables_mapping.items(): - - if key in config_variables: - # passed in & .env & parameters - continue - else: - # config variables - parsed_value = parse_data( - value, - config_variables, - functions - ) - config_variables[key] = parsed_value - - testcase_dict["config"]["variables"] = config_variables - - # parse config name - testcase_dict["config"]["name"] = parse_data( - testcase_dict["config"].get("name", ""), - config_variables, - functions - ) - - # parse config request - testcase_dict["config"]["request"] = parse_data( - testcase_dict["config"].get("request", {}), - config_variables, - functions - ) - - # put loaded project functions to config - testcase_dict["config"]["functions"] = functions - parsed_testcases_list.append(testcase_dict) - - # unset OS environment variables - utils.unset_os_environ(env_mapping) - - return parsed_testcases_list + # unset OS environment variables + utils.unset_os_environ(env_mapping) diff --git a/httprunner/runner.py b/httprunner/runner.py index bcdda8bd..e38aa7aa 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -4,31 +4,62 @@ from unittest.case import SkipTest from httprunner import exceptions, logger, response, utils from httprunner.client import HttpSession -from httprunner.compat import OrderedDict -from httprunner.context import Context +from httprunner.context import SessionContext class Runner(object): + """ Running testcases. + + Examples: + >>> functions={...} + >>> config = { + "name": "XXXX", + "base_url": "http://127.0.0.1", + "verify": False + } + >>> runner = Runner(config, functions) + + >>> teststep = { + "name": "teststep description", + "variables": [], # optional + "request": { + "url": "http://127.0.0.1:5000/api/users/1000", + "method": "GET" + } + } + >>> runner.run_test(teststep) + + """ + + def __init__(self, config, functions, http_client_session=None): + """ run testcase or testsuite. + + Args: + config (dict): testcase/testsuite config dict + + { + "name": "ABC", + "variables": {}, + "setup_hooks", [], + "teardown_hooks", [] + } + + http_client_session (instance): requests.Session(), or locust.client.Session() instance. - def __init__(self, config_dict=None, http_client_session=None): - """ Each testcase is corresponding to one Runner() instance - and one Context() instance. """ - self.http_client_session = http_client_session - config_dict = config_dict or {} + base_url = config.get("base_url") + self.verify = config.get("verify", True) + self.output = config.get("output", []) + self.functions = functions self.evaluated_validators = [] - # testcase variables - config_variables = config_dict.get("variables", {}) - # testcase functions - config_functions = config_dict.get("functions", {}) # testcase setup hooks - testcase_setup_hooks = config_dict.pop("setup_hooks", []) + testcase_setup_hooks = config.get("setup_hooks", []) # testcase teardown hooks - self.testcase_teardown_hooks = config_dict.pop("teardown_hooks", []) + self.testcase_teardown_hooks = config.get("teardown_hooks", []) - self.context = Context(config_variables, config_functions) - self.init_test(config_dict, "testcase") + self.http_client_session = http_client_session or HttpSession(base_url) + self.session_context = SessionContext(self.functions) if testcase_setup_hooks: self.do_hook_actions(testcase_setup_hooks) @@ -37,57 +68,6 @@ class Runner(object): if self.testcase_teardown_hooks: self.do_hook_actions(self.testcase_teardown_hooks) - def init_test(self, test_dict, level): - """ create/update context variables binds - - Args: - test_dict (dict): - level (enum): "testcase" or "teststep" - testcase: - { - "name": "testcase description", - "variables": [], # optional - "request": { - "base_url": "http://127.0.0.1:5000", - "headers": { - "User-Agent": "iOS/2.8.3" - } - } - } - teststep: - { - "name": "teststep description", - "variables": [], # optional - "request": { - "url": "/api/get-token", - "method": "POST", - "headers": { - "Content-Type": "application/json" - } - }, - "json": { - "sign": "f1219719911caae89ccc301679857ebfda115ca2" - } - } - - Returns: - dict: parsed request dict - - """ - test_dict = utils.lower_test_dict_keys(test_dict) - - self.context.init_context_variables(level) - variables = test_dict.get('variables', OrderedDict()) - self.context.update_context_variables(variables, level) - - request_config = test_dict.get('request', {}) - parsed_request = self.context.get_parsed_request(request_config, level) - - base_url = parsed_request.pop("base_url", None) - self.http_client_session = self.http_client_session or HttpSession(base_url) - - return parsed_request - def _handle_skip_feature(self, teststep_dict): """ handle skip feature for teststep - skip: skip current test unconditionally @@ -109,12 +89,12 @@ class Runner(object): elif "skipIf" in teststep_dict: skip_if_condition = teststep_dict["skipIf"] - if self.context.eval_content(skip_if_condition): + if self.session_context.eval_content(skip_if_condition): skip_reason = "{} evaluate to True".format(skip_if_condition) elif "skipUnless" in teststep_dict: skip_unless_condition = teststep_dict["skipUnless"] - if not self.context.eval_content(skip_unless_condition): + if not self.session_context.eval_content(skip_unless_condition): skip_reason = "{} evaluate to False".format(skip_unless_condition) if skip_reason: @@ -124,9 +104,9 @@ class Runner(object): for action in actions: logger.log_debug("call hook: {}".format(action)) # TODO: check hook function if valid - self.context.eval_content(action) + self.session_context.eval_content(action) - def run_test(self, teststep_dict): + def _run_teststep(self, teststep_dict): """ run single teststep. Args: @@ -135,7 +115,7 @@ class Runner(object): "name": "teststep description", "skip": "skip this test unconditionally", "times": 3, - "variables": [], # optional, override + "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", @@ -162,10 +142,14 @@ class Runner(object): self._handle_skip_feature(teststep_dict) # prepare - extractors = teststep_dict.get("extract", []) - validators = teststep_dict.get("validate", []) - parsed_request = self.init_test(teststep_dict, level="teststep") - self.context.update_teststep_variables_mapping("request", parsed_request) + teststep_dict = utils.lower_test_dict_keys(teststep_dict) + teststep_variables = teststep_dict.get("variables", {}) + self.session_context.init_teststep_variables(teststep_variables) + + # parse teststep request + raw_request = teststep_dict.get('request', {}) + parsed_teststep_request = self.session_context.eval_content(raw_request) + self.session_context.update_teststep_variables("request", parsed_teststep_request) # setup hooks setup_hooks = teststep_dict.get("setup_hooks", []) @@ -173,9 +157,10 @@ class Runner(object): self.do_hook_actions(setup_hooks) try: - url = parsed_request.pop('url') - method = parsed_request.pop('method') - group_name = parsed_request.pop("group", None) + url = parsed_teststep_request.pop('url') + method = parsed_teststep_request.pop('method') + parsed_teststep_request.setdefault("verify", self.verify) + group_name = parsed_teststep_request.pop("group", None) except KeyError: raise exceptions.ParamsError("URL or METHOD missed!") @@ -188,14 +173,14 @@ class Runner(object): raise exceptions.ParamsError(err_msg) logger.log_info("{method} {url}".format(method=method, url=url)) - logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_request)) + logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_teststep_request)) # request resp = self.http_client_session.request( method, url, name=group_name, - **parsed_request + **parsed_teststep_request ) resp_obj = response.ResponseObject(resp) @@ -203,21 +188,23 @@ class Runner(object): teardown_hooks = teststep_dict.get("teardown_hooks", []) if teardown_hooks: logger.log_info("start to run teardown hooks") - self.context.update_teststep_variables_mapping("response", resp_obj) + self.session_context.update_teststep_variables("response", resp_obj) self.do_hook_actions(teardown_hooks) # extract + extractors = teststep_dict.get("extract", []) extracted_variables_mapping = resp_obj.extract_response(extractors) - self.context.update_testcase_runtime_variables_mapping(extracted_variables_mapping) + self.session_context.update_seesion_variables(extracted_variables_mapping) # validate + validators = teststep_dict.get("validate", []) try: - self.evaluated_validators = self.context.validate(validators, resp_obj) + self.evaluated_validators = self.session_context.validate(validators, resp_obj) except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure): # log request err_req_msg = "request: \n" - err_req_msg += "headers: {}\n".format(parsed_request.pop("headers", {})) - for k, v in parsed_request.items(): + err_req_msg += "headers: {}\n".format(parsed_teststep_request.pop("headers", {})) + for k, v in parsed_teststep_request.items(): err_req_msg += "{}: {}\n".format(k, repr(v)) logger.log_error(err_req_msg) @@ -230,10 +217,66 @@ class Runner(object): raise + def _run_testcase(self, testcase_dict): + """ run single testcase. + """ + config = testcase_dict.get("config", {}) + test_runner = Runner(config, self.functions, self.http_client_session) + + teststeps = testcase_dict.get("teststeps", []) + for index, teststep_dict in enumerate(teststeps): + test_runner.run_test(teststep_dict) + + self.session_context.update_seesion_variables(test_runner.extract_sessions()) + + def run_test(self, teststep_dict): + """ run single teststep of testcase. + teststep_dict may be in 3 types. + + Args: + teststep_dict (dict): + + # teststep + { + "name": "teststep description", + "variables": [], # optional + "request": { + "url": "http://127.0.0.1:5000/api/users/1000", + "method": "GET" + } + } + + # embeded testcase + { + "config": {...}, + "teststeps": [ + {...}, + {...} + ] + } + + # TODO: function + { + "name": "exec function", + "function": "${func()}" + } + + """ + if "config" in teststep_dict: + self._run_testcase(teststep_dict) + else: + # api + self._run_teststep(teststep_dict) + + def extract_sessions(self): + """ + """ + return self.extract_output(self.output) + def extract_output(self, output_variables_list): """ extract output variables """ - variables_mapping = self.context.teststep_variables_mapping + variables_mapping = self.session_context.session_variables_mapping output = {} for variable in output_variables_list: diff --git a/httprunner/utils.py b/httprunner/utils.py index 5fd35d8f..dfea2059 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -268,6 +268,113 @@ def ensure_mapping_format(variables): return variables_ordered_dict +def _convert_validators_to_mapping(validators): + """ convert validators list to mapping. + + Args: + validators (list): validators in list + + Returns: + dict: validators mapping, use (check, comparator) as key. + + Examples: + >>> validators = [ + {"check": "v1", "expect": 201, "comparator": "eq"}, + {"check": {"b": 1}, "expect": 200, "comparator": "eq"} + ] + >>> _convert_validators_to_mapping(validators) + { + ("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"}, + ('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"} + } + + """ + validators_mapping = {} + + for validator in validators: + if not isinstance(validator["check"], collections.Hashable): + check = json.dumps(validator["check"]) + else: + check = validator["check"] + + key = (check, validator["comparator"]) + validators_mapping[key] = validator + + return validators_mapping + + +def extend_validators(raw_validators, override_validators): + """ extend raw_validators with override_validators. + override_validators will merge and override raw_validators. + + Args: + raw_validators (dict): + override_validators (dict): + + Returns: + list: extended validators + + Examples: + >>> raw_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] + >>> override_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] + >>> extend_validators(raw_validators, override_validators) + [ + {"check": "v1", "expect": 201, "comparator": "eq"}, + {"check": "s2", "expect": 16, "comparator": "len_eq"}, + {"check": "s3", "expect": 12, "comparator": "len_eq"} + ] + + """ + + if not raw_validators: + return override_validators + + elif not override_validators: + return raw_validators + + else: + def_validators_mapping = _convert_validators_to_mapping(raw_validators) + ref_validators_mapping = _convert_validators_to_mapping(override_validators) + + def_validators_mapping.update(ref_validators_mapping) + return list(def_validators_mapping.values()) + + +def extend_variables(raw_variables, override_variables): + """ extend raw_variables with override_variables. + override_variables will merge and override raw_variables. + + Args: + raw_variables (list): + override_variables (list): + + Returns: + OrderedDict: extended variables mapping + + Examples: + >>> raw_variables = [{"var1": "val1"}, {"var2": "val2"}] + >>> override_variables = [{"var1": "val111"}, {"var3": "val3"}] + >>> extend_variables(raw_variables, override_variables) + OrderedDict([ + ('var1', 'val111'), + ('var2', 'val2'), + ('var3', 'val3') + ]) + + """ + if not raw_variables: + return override_variables + + elif not override_variables: + return raw_variables + + else: + raw_variables_mapping = ensure_mapping_format(raw_variables) + override_variables_mapping = ensure_mapping_format(override_variables) + raw_variables_mapping.update(override_variables_mapping) + return raw_variables_mapping + + def get_testcase_io(testcase): """ get testcase input(variables) and output. diff --git a/httprunner/validator.py b/httprunner/validator.py index 54d7ffc9..70b44eeb 100644 --- a/httprunner/validator.py +++ b/httprunner/validator.py @@ -54,21 +54,50 @@ def is_testcases(data_structure): Args: data_structure (dict): testcase(s) should always be in the following data structure: + { + "project_mapping": { + "PWD": "XXXXX", + "functions": {}, + "env": {} + }, + "testcases": [ + { # testcase data structure + "config": { + "name": "desc1", + "path": "testcase1_path", + "variables": [], # optional + }, + "teststeps": [ + # teststep data structure + { + 'name': 'test step desc1', + 'variables': [], # optional + 'extract': [], # optional + 'validate': [], + 'request': {} + }, + teststep2 # another teststep dict + ] + }, + testcase_dict_2 # another testcase dict + ] + } - testcase_dict - or - [ - testcase_dict_1, - testcase_dict_2 - ] Returns: bool: True if data_structure is valid testcase(s), otherwise False. """ - if not isinstance(data_structure, list): - return is_testcase(data_structure) + if not isinstance(data_structure, dict): + return False - for item in data_structure: + if "testcases" not in data_structure: + return False + + testcases = data_structure["testcases"] + if not isinstance(testcases, list): + return False + + for item in testcases: if not is_testcase(item): return False diff --git a/tests/api/basic.yml b/tests/api/basic.yml index 5009a1c8..d91c3301 100644 --- a/tests/api/basic.yml +++ b/tests/api/basic.yml @@ -1,5 +1,11 @@ - api: - def: get_token($user_agent, $device_sn, $os_platform, $app_version) + id: get_token + name: get token + variables: + - user_agent: XXX + - device_sn: API_XXX + - os_platform: XXX + - app_version: XXX request: url: /api/get-token method: POST @@ -8,6 +14,8 @@ device_sn: $device_sn os_platform: $os_platform app_version: $app_version + Content-Type: "application/json" + device_sn: $device_sn json: sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)} validate: @@ -16,15 +24,18 @@ - contains: [{"a": 1, "b": 2}, "a"] - api: - def: create_user($uid, $user_name, $user_password, $token) + id: create_user variables: - user_name: user0 - user_password: "000000" - - uid: 2000 + - uid: 9000 + - token: XXX request: url: /api/users/$uid method: POST headers: + Content-Type: "application/json" + device_sn: $device_sn token: $token json: name: $user_name @@ -33,21 +44,33 @@ - eq: ["status_code", 201] - api: - def: get_user($uid, $token) + id: get_user + variables: + - uid: 9000 + - token: XXX request: url: /api/users/$uid method: GET headers: + Content-Type: "application/json" + device_sn: $device_sn token: $token validate: - eq: ["status_code", 200] - api: - def: update_user($uid, $user_name, $user_password, $token) + id: update_user + variables: + - user_name: user0 + - user_password: "000000" + - uid: 9000 + - token: XXX request: url: /api/users/$uid method: PUT headers: + Content-Type: "application/json" + device_sn: $device_sn token: $token json: name: $user_name @@ -56,40 +79,58 @@ - eq: ["status_code", 200] - api: - def: delete_user($uid, $token) + id: delete_user + variables: + - uid: 9000 + - token: XXX request: url: /api/users/$uid method: DELETE headers: + Content-Type: "application/json" + device_sn: $device_sn token: $token validate: - eq: ["status_code", 200] - api: - def: get_users($token) + id: get_users + variables: + - token: XXX request: url: /api/users method: GET headers: + Content-Type: "application/json" + device_sn: $device_sn token: $token validate: - eq: ["status_code", 200] - api: - def: reset_all($token) + id: reset_all + variables: + - token: XXX request: url: /api/reset-all method: GET headers: + Content-Type: "application/json" + device_sn: $device_sn token: $token validate: - eq: ["status_code", 200] - eq: ["content.success", true] - api: - def: get_headers($n_secs) + id: get_headers + variables: + - n_secs: 1 request: url: /headers + headers: + Content-Type: "application/json" + device_sn: $device_sn method: GET setup_hooks: - ${setup_hook_add_kwargs($request)} diff --git a/tests/data/bugfix_parameters.yml b/tests/data/bugfix_parameters.yml index e81dee42..98a24c95 100644 --- a/tests/data/bugfix_parameters.yml +++ b/tests/data/bugfix_parameters.yml @@ -18,7 +18,7 @@ - test: name: get token with $user_agent, $app_version - api: get_token($user_agent, $device_sn, $os_platform, $app_version) + api: get_token extract: - token: content.token validate: diff --git a/tests/data/demo_parameters.yml b/tests/data/demo_parameters.yml index 4b8eb66c..d43278b5 100644 --- a/tests/data/demo_parameters.yml +++ b/tests/data/demo_parameters.yml @@ -6,20 +6,18 @@ - ["test1","111111"] - ["test2","222222"] variables: + - username: test1 + - user_agent: "iOS/10.1" - device_sn: ${gen_random_string(15)} - os_platform: 'ios' - app_version: 2.8.5 - request: - base_url: ${get_base_url()} - headers: - Content-Type: application/json - device_sn: $device_sn + base_url: ${get_base_url()} output: - token - test: name: get token with $user_agent and $username - api: get_token($user_agent, $device_sn, $os_platform, $app_version) + api: get_token extract: - token: content.token validate: diff --git a/tests/data/demo_testcase.yml b/tests/data/demo_testcase.yml index e0f15764..00fc96c9 100644 --- a/tests/data/demo_testcase.yml +++ b/tests/data/demo_testcase.yml @@ -4,12 +4,11 @@ - var_a: 0 - var_c: "${sum_two(1, 2)}" - PROJECT_KEY: ${ENV(PROJECT_KEY)} - parameters: - - "var_a-var_b": - - [11, 21] - - [12, 22] - - "app_version": "${gen_app_version()}" - request: ${get_default_request()} + # parameters: + # - "var_a-var_b": + # - [11, 21] + # - [12, 22] + # - "app_version": "${gen_app_version()}" - test: name: testcase1-$var_a diff --git a/tests/data/demo_testcase_functions.yml b/tests/data/demo_testcase_functions.yml index 7d69d8be..48e92f1f 100644 --- a/tests/data/demo_testcase_functions.yml +++ b/tests/data/demo_testcase_functions.yml @@ -5,11 +5,7 @@ - device_sn: ${gen_random_string(15)} - os_platform: 'ios' - app_version: '2.8.6' - request: - base_url: ${get_base_url()} - headers: - Content-Type: application/json - device_sn: $device_sn + base_url: ${get_base_url()} - test: name: get token @@ -38,6 +34,8 @@ url: /api/users/1000 method: POST headers: + Content-Type: application/json + device_sn: $device_sn token: $token json: name: $user_name @@ -52,6 +50,8 @@ url: /api/users/1000 method: POST headers: + Content-Type: application/json + device_sn: $device_sn token: $token json: name: "user1" diff --git a/tests/data/demo_testcase_layer.yml b/tests/data/demo_testcase_layer.yml index aed22861..49c2b0bc 100644 --- a/tests/data/demo_testcase_layer.yml +++ b/tests/data/demo_testcase_layer.yml @@ -5,17 +5,13 @@ - device_sn: ${gen_random_string(15)} - os_platform: 'ios' - app_version: '2.8.6' - request: - base_url: ${get_base_url()} - headers: - Content-Type: application/json - device_sn: $device_sn + base_url: ${get_base_url()} output: - token - test: name: get token with $user_agent, $app_version - api: get_token($user_agent, $device_sn, $os_platform, $app_version) + api: get_token extract: - token: content.token validate: @@ -25,14 +21,19 @@ - test: name: reset all users - api: reset_all($token) + api: reset_all + variables: + - token: $token validate: - {"check": "status_code", "expect": 200} - {"check": "content.success", "expect": true} - test: name: get user that does not exist - api: get_user(1000, $token) + api: get_user + variables: + - uid: 1000 + - token: $token validate: - {"check": "status_code", "expect": 404} - {"check": "content.success", "expect": false} @@ -40,16 +41,21 @@ - test: name: create user which does not exist variables: + - uid: 1000 - user_name: "user1" - user_password: "123456" - api: create_user(1000, $user_name, $user_password, $token) + - token: $token + api: create_user validate: - {"check": "status_code", "expect": 201} - {"check": "content.success", "expect": true} - test: name: get user that has been created - api: get_user(1000, $token) + api: get_user + variables: + - uid: 1000 + - token: $token validate: - {"check": "status_code", "expect": 200} - {"check": "content.success", "expect": true} @@ -58,9 +64,11 @@ - test: name: create user which exists variables: + - uid: 1000 - user_name: "user1" - user_password: "123456" - api: create_user(1000, $user_name, $user_password, $token) + - token: $token + api: create_user validate: - {"check": "status_code", "expect": 500} - {"check": "content.success", "expect": false} @@ -68,16 +76,21 @@ - test: name: update user which exists variables: + - uid: 1000 - user_name: "user1" - user_password: "654321" - api: update_user(1000, $user_name, $user_password, $token) + - token: $token + api: update_user validate: - {"check": "status_code", "expect": 200} - {"check": "content.success", "expect": true} - test: name: get user that has been updated - api: get_user(1000, $token) + api: get_user + variables: + - uid: 1000 + - token: $token validate: - {"check": "status_code", "expect": 200} - {"check": "content.success", "expect": true} @@ -85,21 +98,28 @@ - test: name: get users - api: get_users($token) + api: get_users + variables: + - token: $token validate: - {"check": "status_code", "expect": 200} - {"check": "content.count", "expect": 1} - test: name: delete user that exists - api: delete_user(1000, $token) + api: delete_user + variables: + - uid: 1000 + - token: $token validate: - {"check": "status_code", "expect": 200} - {"check": "content.success", "expect": true} - test: name: get users - api: get_users($token) + api: get_users + variables: + - token: $token validate: - {"check": "status_code", "expect": 200} - {"check": "content.count", "expect": 0} @@ -107,16 +127,20 @@ - test: name: create user which has been deleted variables: + - uid: 1000 - user_name: "user1" - user_password: "123456" - api: create_user(1000, $user_name, $user_password, $token) + - token: $token + api: create_user validate: - {"check": "status_code", "expect": 201} - {"check": "content.success", "expect": true} - test: name: get users - api: get_users($token) + api: get_users + variables: + - token: $token validate: - {"check": "status_code", "expect": 200} - {"check": "content.count", "expect": 1} diff --git a/tests/data/demo_testcase_variables.yml b/tests/data/demo_testcase_variables.yml index 2fe15ae2..e9ec4925 100644 --- a/tests/data/demo_testcase_variables.yml +++ b/tests/data/demo_testcase_variables.yml @@ -2,11 +2,7 @@ name: "create user testcases." variables: - device_sn: 'HZfFBh6tU59EdXJ' - request: - base_url: ${get_base_url()} - headers: - Content-Type: application/json - device_sn: $device_sn + base_url: ${get_base_url()} - test: name: get token @@ -41,6 +37,8 @@ url: /api/users/1000 method: POST headers: + Content-Type: application/json + device_sn: $device_sn token: $token json: name: $user_name @@ -55,6 +53,8 @@ url: /api/users/1000 method: POST headers: + Content-Type: application/json + device_sn: $device_sn token: $token json: name: "user1" diff --git a/tests/httpbin/basic.yml b/tests/httpbin/basic.yml index f8eacca2..b3f9aca9 100644 --- a/tests/httpbin/basic.yml +++ b/tests/httpbin/basic.yml @@ -1,7 +1,6 @@ - config: name: basic test with httpbin - request: - base_url: https://httpbin.org/ + base_url: https://httpbin.org/ - test: name: index diff --git a/tests/httpbin/hooks.yml b/tests/httpbin/hooks.yml index b7e8e8ef..e4670772 100644 --- a/tests/httpbin/hooks.yml +++ b/tests/httpbin/hooks.yml @@ -1,7 +1,6 @@ - config: name: basic test with httpbin - request: - base_url: ${get_httpbin_server()} + base_url: ${get_httpbin_server()} setup_hooks: - ${hook_print(setup)} teardown_hooks: diff --git a/tests/httpbin/load_image.yml b/tests/httpbin/load_image.yml index 8ada7f21..1243ef86 100644 --- a/tests/httpbin/load_image.yml +++ b/tests/httpbin/load_image.yml @@ -1,7 +1,6 @@ - config: name: load images - request: - base_url: ${get_httpbin_server()} + base_url: ${get_httpbin_server()} - test: name: get png image diff --git a/tests/httpbin/upload.yml b/tests/httpbin/upload.yml index 38d64a93..3ac423c5 100644 --- a/tests/httpbin/upload.yml +++ b/tests/httpbin/upload.yml @@ -1,7 +1,6 @@ - config: name: test upload file with httpbin - request: - base_url: ${get_httpbin_server()} + base_url: ${get_httpbin_server()} - test: name: upload file diff --git a/tests/test_api.py b/tests/test_api.py index 85a3d86a..3a1cea67 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -18,7 +18,7 @@ class TestHttpRunner(ApiServerUnittest): os.path.join( os.getcwd(), 'tests/data/demo_testcase_hardcode.json') ] - self.testcases = [{ + testcases = [{ 'config': { 'name': 'testcase description', 'request': { @@ -61,6 +61,9 @@ class TestHttpRunner(ApiServerUnittest): } ] }] + self.tests_mapping = { + "testcases": testcases + } self.reset_all() def reset_all(self): @@ -89,7 +92,7 @@ class TestHttpRunner(ApiServerUnittest): shutil.rmtree(report_save_dir) def test_run_testcases(self): - runner = HttpRunner().run(self.testcases) + runner = HttpRunner().run(self.tests_mapping) summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 2) @@ -133,7 +136,10 @@ class TestHttpRunner(ApiServerUnittest): ] } ] - runner = HttpRunner().run(testcases) + tests_mapping = { + "testcases": testcases + } + runner = HttpRunner().run(tests_mapping) summary = runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 1) @@ -148,11 +154,18 @@ class TestHttpRunner(ApiServerUnittest): report_save_dir = os.path.join(os.getcwd(), 'reports', output_folder_name) shutil.rmtree(report_save_dir) - def test_testcase_layer(self): - runner = HttpRunner(failfast=True).run("tests/testcases/smoketest.yml") + def test_testcase_layer_with_api(self): + runner = HttpRunner(failfast=True).run("tests/testcases/setup.yml") summary = runner.summary self.assertTrue(summary["success"]) - self.assertEqual(summary["stat"]["testsRun"], 8) + self.assertEqual(summary["details"][0]["records"][0]["name"], "get token (setup)") + self.assertEqual(summary["stat"]["testsRun"], 2) + + def test_testcase_layer_with_testcase(self): + runner = HttpRunner(failfast=True).run("tests/testsuites/create_users.yml") + summary = runner.summary + self.assertTrue(summary["success"]) + self.assertEqual(summary["stat"]["testsRun"], 2) def test_run_httprunner_with_hooks(self): testcase_file_path = os.path.join( @@ -165,11 +178,9 @@ class TestHttpRunner(ApiServerUnittest): self.assertLess(end_time - start_time, 60) def test_run_httprunner_with_teardown_hooks_alter_response(self): - config = {"name": "test teardown hooks"} - config.update(loader.load_project_tests("tests")) testcases = [ { - "config": config, + "config": {"name": "test teardown hooks"}, "teststeps": [ { "name": "test teardown hooks", @@ -195,7 +206,12 @@ class TestHttpRunner(ApiServerUnittest): ] } ] - runner = HttpRunner().run(testcases) + loader.load_project_tests("tests") + tests_mapping = { + "project_mapping": loader.project_mapping, + "testcases": testcases + } + runner = HttpRunner().run(tests_mapping) summary = runner.summary self.assertTrue(summary["success"]) @@ -223,7 +239,12 @@ class TestHttpRunner(ApiServerUnittest): ] } ] - runner = HttpRunner().run(testcases) + loader.load_project_tests("tests") + tests_mapping = { + "project_mapping": loader.project_mapping, + "testcases": testcases + } + runner = HttpRunner().run(tests_mapping) summary = runner.summary self.assertFalse(summary["success"]) self.assertEqual(summary["stat"]["errors"], 1) @@ -249,7 +270,12 @@ class TestHttpRunner(ApiServerUnittest): ] } ] - runner = HttpRunner().run(testcases) + loader.load_project_tests("tests") + tests_mapping = { + "project_mapping": loader.project_mapping, + "testcases": testcases + } + runner = HttpRunner().run(tests_mapping) summary = runner.summary self.assertFalse(summary["success"]) self.assertEqual(summary["stat"]["errors"], 1) @@ -262,14 +288,6 @@ class TestHttpRunner(ApiServerUnittest): self.assertEqual(summary["stat"]["testsRun"], 3) self.assertEqual(summary["stat"]["successes"], 3) - def test_run_testcases_hardcode(self): - runner = HttpRunner().run(self.testcase_file_path_list) - summary = runner.summary - self.assertTrue(summary["success"]) - self.assertTrue(summary["success"]) - self.assertEqual(summary["stat"]["testsRun"], 6) - self.assertEqual(summary["stat"]["successes"], 6) - def test_run_testcase_template_variables(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_variables.yml') @@ -299,7 +317,8 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertIn("token", summary["details"][0]["in_out"]["out"]) - self.assertIn("user_agent", summary["details"][0]["in_out"]["in"]) + # TODO: add + # self.assertIn("user_agent", summary["details"][0]["in_out"]["in"]) def test_run_testcase_with_variables_mapping(self): testcase_file_path = os.path.join( @@ -311,66 +330,238 @@ class TestHttpRunner(ApiServerUnittest): summary = runner.summary self.assertTrue(summary["success"]) self.assertIn("token", summary["details"][0]["in_out"]["out"]) - self.assertGreater(len(summary["details"][0]["in_out"]["in"]), 3) + # TODO: add + # self.assertGreater(len(summary["details"][0]["in_out"]["in"]), 3) def test_run_testcase_with_parameters(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_parameters.yml') runner = HttpRunner().run(testcase_file_path) summary = runner.summary - self.assertEqual( - summary["details"][0]["in_out"]["in"]["user_agent"], - "iOS/10.1" - ) - self.assertEqual( - summary["details"][2]["in_out"]["in"]["user_agent"], - "iOS/10.2" - ) - self.assertEqual( - summary["details"][4]["in_out"]["in"]["user_agent"], - "iOS/10.3" - ) + # TODO: add parameterize + # self.assertEqual( + # summary["details"][0]["in_out"]["in"]["user_agent"], + # "iOS/10.1" + # ) + # self.assertEqual( + # summary["details"][2]["in_out"]["in"]["user_agent"], + # "iOS/10.2" + # ) + # self.assertEqual( + # summary["details"][4]["in_out"]["in"]["user_agent"], + # "iOS/10.3" + # ) self.assertTrue(summary["success"]) - self.assertEqual(len(summary["details"]), 3 * 2) - self.assertEqual(summary["stat"]["testsRun"], 3 * 2) + self.assertEqual(len(summary["details"]), 1) + # self.assertEqual(len(summary["details"]), 3 * 2) + # self.assertEqual(summary["stat"]["testsRun"], 3 * 2) self.assertIn("in", summary["details"][0]["in_out"]) self.assertIn("out", summary["details"][0]["in_out"]) def test_run_testcase_with_parameters_name(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_parameters.yml') - testcases = loader.load_tests(testcase_file_path) - parsed_testcases = parser.parse_tests(testcases) + tests_mapping = loader.load_tests(testcase_file_path) + parser.parse_tests(tests_mapping) runner = HttpRunner() - test_suite = runner._add_tests(parsed_testcases) + test_suite = runner._add_tests(tests_mapping) self.assertEqual( test_suite._tests[0].teststeps[0]['name'], 'get token with iOS/10.1 and test1' ) - self.assertEqual( - test_suite._tests[1].teststeps[0]['name'], - 'get token with iOS/10.1 and test2' - ) - self.assertEqual( - test_suite._tests[2].teststeps[0]['name'], - 'get token with iOS/10.2 and test1' - ) - self.assertEqual( - test_suite._tests[3].teststeps[0]['name'], - 'get token with iOS/10.2 and test2' - ) - self.assertEqual( - test_suite._tests[4].teststeps[0]['name'], - 'get token with iOS/10.3 and test1' - ) - self.assertEqual( - test_suite._tests[5].teststeps[0]['name'], - 'get token with iOS/10.3 and test2' - ) + # TODO: add parameterize + # self.assertEqual( + # test_suite._tests[1].teststeps[0]['name'], + # 'get token with iOS/10.1 and test2' + # ) + # self.assertEqual( + # test_suite._tests[2].teststeps[0]['name'], + # 'get token with iOS/10.2 and test1' + # ) + # self.assertEqual( + # test_suite._tests[3].teststeps[0]['name'], + # 'get token with iOS/10.2 and test2' + # ) + # self.assertEqual( + # test_suite._tests[4].teststeps[0]['name'], + # 'get token with iOS/10.3 and test1' + # ) + # self.assertEqual( + # test_suite._tests[5].teststeps[0]['name'], + # 'get token with iOS/10.3 and test2' + # ) def test_validate_response_content(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/httpbin/basic.yml') runner = HttpRunner().run(testcase_file_path) self.assertTrue(runner.summary["success"]) + + +class TestApi(ApiServerUnittest): + + def test_testcase_loader(self): + testcase_path = "tests/testcases/setup.yml" + tests_mapping = loader.load_tests(testcase_path) + + project_mapping = tests_mapping["project_mapping"] + self.assertIsInstance(project_mapping, dict) + self.assertIn("PWD", project_mapping) + self.assertIn("functions", project_mapping) + self.assertIn("env", project_mapping) + + testcases = tests_mapping["testcases"] + self.assertIsInstance(testcases, list) + self.assertEqual(len(testcases), 1) + testcase_config = testcases[0]["config"] + self.assertEqual(testcase_config["name"], "setup and reset all.") + self.assertIn("path", testcase_config) + + testcase_teststeps = testcases[0]["teststeps"] + self.assertEqual(len(testcase_teststeps), 2) + self.assertIn("api", testcase_teststeps[0]) + self.assertEqual(testcase_teststeps[0]["name"], "get token (setup)") + self.assertIsInstance(testcase_teststeps[0]["variables"], list) + self.assertIn("api_def", testcase_teststeps[0]) + self.assertEqual(testcase_teststeps[0]["api_def"]["request"]["url"], "/api/get-token") + + def test_testcase_parser(self): + testcase_path = "tests/testcases/setup.yml" + tests_mapping = loader.load_tests(testcase_path) + + parser.parse_tests(tests_mapping) + parsed_testcases = tests_mapping["testcases"] + + self.assertEqual(len(parsed_testcases), 1) + + self.assertNotIn("variables", parsed_testcases[0]["config"]) + self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2) + + teststep1 = parsed_testcases[0]["teststeps"][0] + self.assertEqual(teststep1["name"], "get token (setup)") + self.assertNotIn("api_def", teststep1) + self.assertEqual(teststep1["variables"]["device_sn"], "TESTCASE_SETUP_XXX") + self.assertEqual(teststep1["request"]["url"], "http://127.0.0.1:5000/api/get-token") + + def test_testcase_add_tests(self): + testcase_path = "tests/testcases/setup.yml" + tests_mapping = loader.load_tests(testcase_path) + + parser.parse_tests(tests_mapping) + runner = HttpRunner() + test_suite = runner._add_tests(tests_mapping) + + self.assertEqual(len(test_suite._tests), 1) + teststeps = test_suite._tests[0].teststeps + self.assertEqual(teststeps[0]["name"], "get token (setup)") + self.assertEqual(teststeps[0]["variables"]["device_sn"], "TESTCASE_SETUP_XXX") + self.assertIn("api", teststeps[0]) + + def test_testcase_simple_run_suite(self): + testcase_path = "tests/testcases/setup.yml" + tests_mapping = loader.load_tests(testcase_path) + parser.parse_tests(tests_mapping) + runner = HttpRunner() + test_suite = runner._add_tests(tests_mapping) + tests_results = runner._run_suite(test_suite) + self.assertEqual(len(tests_results[0][1].records), 2) + + def test_testcase_complex_run_suite(self): + testcase_path = "tests/testcases/create_and_check.yml" + tests_mapping = loader.load_tests(testcase_path) + parser.parse_tests(tests_mapping) + runner = HttpRunner() + test_suite = runner._add_tests(tests_mapping) + tests_results = runner._run_suite(test_suite) + self.assertEqual(len(tests_results[0][1].records), 4) + + results = tests_results[0][1] + self.assertEqual( + results.records[0]["name"], + "setup and reset all (override)." + ) + self.assertEqual( + results.records[1]["name"], + "make sure user 9001 does not exist" + ) + + def test_testsuite_loader(self): + testcase_path = "tests/testsuites/create_users.yml" + tests_mapping = loader.load_tests(testcase_path) + + project_mapping = tests_mapping["project_mapping"] + self.assertIsInstance(project_mapping, dict) + self.assertIn("PWD", project_mapping) + self.assertIn("functions", project_mapping) + self.assertIn("env", project_mapping) + + testcases = tests_mapping["testcases"] + self.assertIsInstance(testcases, list) + self.assertEqual(len(testcases), 1) + testcase_config = testcases[0]["config"] + self.assertEqual(testcase_config["name"], "create users with uid") + self.assertIn("path", testcase_config) + + testcase_teststeps = testcases[0]["teststeps"] + self.assertEqual(len(testcase_teststeps), 2) + self.assertIn("testcase_def", testcase_teststeps[0]) + self.assertEqual(testcase_teststeps[0]["name"], "create user 1000 and check result.") + self.assertIsInstance(testcase_teststeps[0]["testcase_def"], dict) + self.assertEqual(testcase_teststeps[0]["testcase_def"]["config"]["name"], "create user and check result.") + self.assertEqual(len(testcase_teststeps[0]["testcase_def"]["teststeps"]), 4) + self.assertEqual(testcase_teststeps[0]["testcase_def"]["teststeps"][0]["name"], "setup and reset all (override).") + + def test_testsuite_parser(self): + testcase_path = "tests/testsuites/create_users.yml" + tests_mapping = loader.load_tests(testcase_path) + + parser.parse_tests(tests_mapping) + + parsed_testcases = tests_mapping["testcases"] + self.assertEqual(len(parsed_testcases), 1) + self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2) + + testcase1 = parsed_testcases[0]["teststeps"][0] + self.assertEqual(testcase1["config"]["name"], "create user 1000 and check result.") + self.assertNotIn("testcase_def", testcase1) + self.assertEqual(len(testcase1["teststeps"]), 4) + self.assertEqual( + testcase1["teststeps"][0]["teststeps"][0]["request"]["url"], + "http://127.0.0.1:5000/api/get-token" + ) + self.assertEqual(len(testcase1["teststeps"][0]["teststeps"][0]["variables"]["device_sn"]), 15) + + def test_testsuite_add_tests(self): + testcase_path = "tests/testsuites/create_users.yml" + tests_mapping = loader.load_tests(testcase_path) + + parser.parse_tests(tests_mapping) + runner = HttpRunner() + test_suite = runner._add_tests(tests_mapping) + + self.assertEqual(len(test_suite._tests), 1) + teststeps = test_suite._tests[0].teststeps + self.assertEqual(teststeps[0]["config"]["name"], "create user 1000 and check result.") + + def test_testsuite_run_suite(self): + testcase_path = "tests/testsuites/create_users.yml" + tests_mapping = loader.load_tests(testcase_path) + + parser.parse_tests(tests_mapping) + + runner = HttpRunner() + test_suite = runner._add_tests(tests_mapping) + tests_results = runner._run_suite(test_suite) + + self.assertEqual(len(tests_results[0][1].records), 2) + + results = tests_results[0][1] + self.assertEqual( + results.records[0]["name"], + "create user 1000 and check result." + ) + self.assertEqual( + results.records[1]["name"], + "create user 1001 and check result." + ) \ No newline at end of file diff --git a/tests/test_context.py b/tests/test_context.py index 2e2dd192..809b4195 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -2,64 +2,38 @@ import os import time import requests -from httprunner import context, exceptions, loader, response +from httprunner import context, exceptions, loader, response, utils from tests.base import ApiServerUnittest class TestContext(ApiServerUnittest): def setUp(self): - project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) - self.context = context.Context( - variables={"SECRET_KEY": "DebugTalk"}, - functions=project_mapping["functions"] + loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + project_mapping = loader.project_mapping + self.context = context.SessionContext( + functions=project_mapping["functions"], + variables={"SECRET_KEY": "DebugTalk"} ) testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml') self.testcases = loader.load_file(testcase_file_path) def test_init_context_functions(self): - context_functions = self.context.TESTCASE_SHARED_FUNCTIONS_MAPPING + context_functions = self.context.FUNCTIONS_MAPPING self.assertIn("gen_md5", context_functions) - def test_init_context_variables(self): + def test_init_teststep_variables(self): self.assertEqual( - self.context.teststep_variables_mapping["SECRET_KEY"], - "DebugTalk" - ) - self.assertEqual( - self.context.testcase_runtime_variables_mapping["SECRET_KEY"], - "DebugTalk" + self.context.teststep_variables_mapping, + {'SECRET_KEY': 'DebugTalk'} ) - def test_update_context_testcase_level(self): - variables = [ - {"TOKEN": "debugtalk"}, - {"data": '{"name": "user", "password": "123456"}'} - ] - self.context.update_context_variables(variables, "testcase") + def test_update_seesion_variables(self): + self.context.update_seesion_variables({"TOKEN": "debugtalk"}) self.assertEqual( - self.context.teststep_variables_mapping["TOKEN"], + self.context.session_variables_mapping["TOKEN"], "debugtalk" ) - self.assertEqual( - self.context.testcase_runtime_variables_mapping["TOKEN"], - "debugtalk" - ) - - def test_update_context_teststep_level(self): - variables = [ - {"TOKEN": "debugtalk"}, - {"data": '{"name": "user", "password": "123456"}'} - ] - self.context.update_context_variables(variables, "teststep") - self.assertEqual( - self.context.teststep_variables_mapping["TOKEN"], - "debugtalk" - ) - self.assertNotIn( - "TOKEN", - self.context.testcase_runtime_variables_mapping - ) def test_eval_content_functions(self): content = "${sleep_N_secs(1)}" @@ -82,29 +56,6 @@ class TestContext(ApiServerUnittest): # "abcDebugTalkdef" # ) - def test_update_testcase_runtime_variables_mapping(self): - variables = {"abc": 123} - self.context.update_testcase_runtime_variables_mapping(variables) - self.assertEqual( - self.context.testcase_runtime_variables_mapping["abc"], - 123 - ) - self.assertEqual( - self.context.teststep_variables_mapping["abc"], - 123 - ) - - def test_update_teststep_variables_mapping(self): - self.context.update_teststep_variables_mapping("abc", 123) - self.assertEqual( - self.context.teststep_variables_mapping["abc"], - 123 - ) - self.assertNotIn( - "abc", - self.context.testcase_runtime_variables_mapping - ) - def test_get_parsed_request(self): variables = [ {"TOKEN": "debugtalk"}, @@ -112,7 +63,7 @@ class TestContext(ApiServerUnittest): {"data": '{"name": "user", "password": "123456"}'}, {"authorization": "${gen_md5($TOKEN, $data, $random)}"} ] - self.context.update_context_variables(variables, "teststep") + self.context.init_teststep_variables(variables) request = { "url": "http://127.0.0.1:5000/api/users/1000", @@ -125,7 +76,7 @@ class TestContext(ApiServerUnittest): }, "data": "$data" } - parsed_request = self.context.get_parsed_request(request, level="teststep") + parsed_request = self.context.eval_content(request) self.assertIn("authorization", parsed_request["headers"]) self.assertEqual(len(parsed_request["headers"]["authorization"]), 32) self.assertIn("random", parsed_request["headers"]) @@ -159,7 +110,7 @@ class TestContext(ApiServerUnittest): {"resp_status_code": 200}, {"resp_body_success": True} ] - self.context.update_context_variables(variables, "teststep") + self.context.init_teststep_variables(variables) with self.assertRaises(exceptions.ValidationFailure): self.context.validate(validators, resp_obj) @@ -174,7 +125,7 @@ class TestContext(ApiServerUnittest): {"resp_status_code": 201}, {"resp_body_success": True} ] - self.context.update_context_variables(variables, "teststep") + self.context.init_teststep_variables(variables) self.context.validate(validators, resp_obj) def test_validate_exception(self): @@ -188,7 +139,7 @@ class TestContext(ApiServerUnittest): {"check": "$resp_status_code", "comparator": "eq", "expect": 201} ] variables = [] - self.context.update_context_variables(variables, "teststep") + self.context.init_teststep_variables(variables) with self.assertRaises(exceptions.VariableNotFound): self.context.validate(validators, resp_obj) @@ -197,7 +148,7 @@ class TestContext(ApiServerUnittest): variables = [ {"resp_status_code": 200} ] - self.context.update_context_variables(variables, "teststep") + self.context.init_teststep_variables(variables) with self.assertRaises(exceptions.ValidationFailure): self.context.validate(validators, resp_obj) diff --git a/tests/test_loader.py b/tests/test_loader.py index c0ff483b..5630a14a 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -195,10 +195,12 @@ class TestModuleLoader(unittest.TestCase): self.assertNotIn("is_py3", module_functions) def test_load_debugtalk_module(self): - project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "httprunner")) + loader.load_project_tests(os.path.join(os.getcwd(), "httprunner")) + project_mapping = loader.project_mapping self.assertNotIn("alter_response", project_mapping["functions"]) - project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + project_mapping = loader.project_mapping self.assertIn("alter_response", project_mapping["functions"]) is_status_code_200 = project_mapping["functions"]["is_status_code_200"] @@ -230,16 +232,21 @@ class TestModuleLoader(unittest.TestCase): def test_load_tests(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase.yml') - testcases = loader.load_tests(testcase_file_path) + tests_mapping = loader.load_tests(testcase_file_path) + testcases = tests_mapping["testcases"] self.assertIsInstance(testcases, list) - self.assertEqual( - testcases[0]["config"]["request"], - '${get_default_request()}' - ) self.assertEqual(testcases[0]["config"]["name"], '123$var_a') self.assertIn( "sum_two", - testcases[0]["config"]["functions"] + tests_mapping["project_mapping"]["functions"] + ) + self.assertEqual( + testcases[0]["config"]["variables"][1]["var_c"], + "${sum_two(1, 2)}" + ) + self.assertEqual( + testcases[0]["config"]["variables"][2]["PROJECT_KEY"], + "${ENV(PROJECT_KEY)}" ) @@ -247,198 +254,98 @@ class TestSuiteLoader(unittest.TestCase): @classmethod def setUpClass(cls): - cls.project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + cls.project_mapping = loader.project_mapping + cls.tests_def_mapping = loader.tests_def_mapping - def test_load_teststeps(self): - test_block = { + def test_load_teststep_testcase(self): + raw_teststep = { "name": "setup and reset all (override).", - "suite": "setup_and_reset($device_sn)", + "testcase": "testcases/setup.yml", + "variables": [ + {"device_sn": "$device_sn"} + ], "output": ["token", "device_sn"] } - teststeps = loader._load_teststeps(test_block, self.project_mapping) + testcase = loader.load_teststep(raw_teststep) + self.assertEqual( + "setup and reset all (override).", + testcase["name"] + ) + teststeps = testcase["testcase_def"]["teststeps"] self.assertEqual(len(teststeps), 2) - self.assertEqual(teststeps[0]["name"], "get token") + self.assertEqual(teststeps[0]["name"], "get token (setup)") self.assertEqual(teststeps[1]["name"], "reset all users") def test_load_testcase(self): - raw_testcase = loader.load_file("tests/testcases/smoketest.yml") - testcase = loader._load_testcase(raw_testcase, self.project_mapping) - self.assertEqual(testcase["config"]["name"], "smoketest") + raw_testcase = loader.load_file("tests/testsuites/create_users.yml") + testcase = loader.load_testcase(raw_testcase) + self.assertEqual(testcase["config"]["name"], "create users with uid") self.assertIn("device_sn", testcase["config"]["variables"][0]) - self.assertEqual(len(testcase["teststeps"]), 8) - self.assertEqual(testcase["teststeps"][0]["name"], "get token") - - def test_get_block_by_name(self): - ref_call = "get_user($uid, $token)" - block = loader._get_block_by_name(ref_call, "def-api", self.project_mapping) - self.assertEqual(block["request"]["url"], "/api/users/$uid") - self.assertEqual(block["function_meta"]["func_name"], "get_user") - self.assertEqual(block["function_meta"]["args"], ['$uid', '$token']) - - def test_get_block_by_name_args_mismatch(self): - ref_call = "get_user($uid, $token, $var)" - with self.assertRaises(exceptions.ParamsError): - loader._get_block_by_name(ref_call, "def-api", self.project_mapping) - - def test_override_block(self): - def_block = loader._get_block_by_name( - "get_token($user_agent, $device_sn, $os_platform, $app_version)", - "def-api", - self.project_mapping - ) - test_block = { - "name": "override block", - "times": 3, - "variables": [ - {"var": 123} - ], - 'request': { - 'url': '/api/get-token', - 'method': 'POST', - 'headers': {'user_agent': '$user_agent', 'device_sn': '$device_sn', 'os_platform': '$os_platform', 'app_version': '$app_version'}, - 'json': {'sign': '${get_sign($user_agent, $device_sn, $os_platform, $app_version)}'} - }, - 'validate': [ - {'eq': ['status_code', 201]}, - {'len_eq': ['content.token', 32]} - ] - } - - extended_block = loader._extend_block(test_block, def_block) - self.assertEqual(extended_block["name"], "override block") - self.assertIn({'var': 123}, extended_block["variables"]) - self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, extended_block["validate"]) - self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, extended_block["validate"]) - self.assertEqual(extended_block["times"], 3) - - def test_get_test_definition_api(self): - api_def = loader._get_test_definition("get_headers", "def-api", self.project_mapping) - self.assertEqual(api_def["request"]["url"], "/headers") - self.assertEqual(len(api_def["setup_hooks"]), 2) - self.assertEqual(len(api_def["teardown_hooks"]), 1) - - with self.assertRaises(exceptions.ApiNotFound): - loader._get_test_definition("get_token_XXX", "def-api", self.project_mapping) - - def test_get_test_definition_suite(self): - api_def = loader._get_test_definition("create_and_check", "def-testcase", self.project_mapping) - self.assertEqual(api_def["config"]["name"], "create user and check result.") - - with self.assertRaises(exceptions.TestcaseNotFound): - loader._get_test_definition("create_and_check_XXX", "def-testcase", self.project_mapping) - - def test_extend_validators(self): - def_validators = [ - {'eq': ['v1', 200]}, - {"check": "s2", "expect": 16, "comparator": "len_eq"} - ] - current_validators = [ - {"check": "v1", "expect": 201}, - {'len_eq': ['s3', 12]} - ] - - extended_validators = loader._extend_validators(def_validators, current_validators) - self.assertIn( - {"check": "v1", "expect": 201, "comparator": "eq"}, - extended_validators - ) - self.assertIn( - {"check": "s2", "expect": 16, "comparator": "len_eq"}, - extended_validators - ) - self.assertIn( - {"check": "s3", "expect": 12, "comparator": "len_eq"}, - extended_validators - ) - - def test_extend_validators_with_dict(self): - def_validators = [ - {'eq': ["a", {"v": 1}]}, - {'eq': [{"b": 1}, 200]} - ] - current_validators = [ - {'len_eq': ['s3', 12]}, - {'eq': [{"b": 1}, 201]} - ] - - extended_validators = loader._extend_validators(def_validators, current_validators) - self.assertEqual(len(extended_validators), 3) - self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'eq'}, extended_validators) - self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'eq'}, extended_validators) - - def test_extend_variables(self): - def_variables = [{"var1": "val1"}, {"var2": "val2"}] - ref_variables = [{"var1": "val111"}, {"var3": "val3"}] - - extended_variables = loader._extend_variables(def_variables, ref_variables) - self.assertIn( - {"var1": "val111"}, - extended_variables - ) - self.assertIn( - {"var2": "val2"}, - extended_variables - ) - self.assertIn( - {"var3": "val3"}, - extended_variables - ) + self.assertEqual(len(testcase["teststeps"]), 2) + self.assertEqual(testcase["teststeps"][0]["name"], "create user 1000 and check result.") + self.assertEqual(testcase["teststeps"][0]["testcase_def"]["config"]["name"], "create user and check result.") def test_load_testcases_by_path_files(self): - testcases_list = [] - # absolute file path path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_hardcode.json') - testcases_list = loader.load_tests(path) + tests_mapping = loader.load_tests(path) + project_mapping = tests_mapping["project_mapping"] + testcases_list = tests_mapping["testcases"] self.assertEqual(len(testcases_list), 1) self.assertEqual(len(testcases_list[0]["teststeps"]), 3) - self.assertIn("get_sign", testcases_list[0]["config"]["functions"]) + self.assertIn("get_sign", project_mapping["functions"]) # relative file path path = 'tests/data/demo_testcase_hardcode.yml' - testcases_list = loader.load_tests(path) + tests_mapping = loader.load_tests(path) + project_mapping = tests_mapping["project_mapping"] + testcases_list = tests_mapping["testcases"] self.assertEqual(len(testcases_list), 1) self.assertEqual(len(testcases_list[0]["teststeps"]), 3) - self.assertIn("get_sign", testcases_list[0]["config"]["functions"]) + self.assertIn("get_sign", project_mapping["functions"]) - # list/set container with file(s) - path = [ - os.path.join(os.getcwd(), 'tests/data/demo_testcase_hardcode.json'), - 'tests/data/demo_testcase_hardcode.yml' - ] - testcases_list = loader.load_tests(path) - self.assertEqual(len(testcases_list), 2) - self.assertEqual(len(testcases_list[0]["teststeps"]), 3) - self.assertEqual(len(testcases_list[1]["teststeps"]), 3) - testcases_list.extend(testcases_list) - self.assertEqual(len(testcases_list), 4) + # TODO: list/set container with file(s) + # path = [ + # os.path.join(os.getcwd(), 'tests/data/demo_testcase_hardcode.json'), + # 'tests/data/demo_testcase_hardcode.yml' + # ] + # testcases_list = loader.load_tests(path) + # self.assertEqual(len(testcases_list), 2) + # self.assertEqual(len(testcases_list[0]["teststeps"]), 3) + # self.assertEqual(len(testcases_list[1]["teststeps"]), 3) + # testcases_list.extend(testcases_list) + # self.assertEqual(len(testcases_list), 4) - for testcase in testcases_list: - for teststep in testcase["teststeps"]: - self.assertIn('name', teststep) - self.assertIn('request', teststep) - self.assertIn('url', teststep['request']) - self.assertIn('method', teststep['request']) + # for testcase in testcases_list: + # for teststep in testcase["teststeps"]: + # self.assertIn('name', teststep) + # self.assertIn('request', teststep) + # self.assertIn('url', teststep['request']) + # self.assertIn('method', teststep['request']) def test_load_testcases_by_path_folder(self): # absolute folder path path = os.path.join(os.getcwd(), 'tests/data') - testcase_list_1 = loader.load_tests(path) + tests_mapping = loader.load_tests(path) + testcase_list_1 = tests_mapping["testcases"] self.assertGreater(len(testcase_list_1), 4) # relative folder path path = 'tests/data/' - testcase_list_2 = loader.load_tests(path) + tests_mapping = loader.load_tests(path) + testcase_list_2 = tests_mapping["testcases"] self.assertEqual(len(testcase_list_1), len(testcase_list_2)) - # list/set container with file(s) - path = [ - os.path.join(os.getcwd(), 'tests/data'), - 'tests/data/' - ] - testcase_list_3 = loader.load_tests(path) - self.assertEqual(len(testcase_list_3), 2 * len(testcase_list_1)) + # TODO: list/set container with file(s) + # path = [ + # os.path.join(os.getcwd(), 'tests/data'), + # 'tests/data/' + # ] + # tests_mapping = loader.load_tests(path) + # testcase_list_3 = tests_mapping["testcases"] + # self.assertEqual(len(testcase_list_3), 2 * len(testcase_list_1)) def test_load_testcases_by_path_not_exist(self): # absolute folder path @@ -451,34 +358,68 @@ class TestSuiteLoader(unittest.TestCase): with self.assertRaises(exceptions.FileNotFound): loader.load_tests(path) - # list/set container with file(s) - path = [ - os.path.join(os.getcwd(), 'tests/data_not_exist'), - 'tests/data_not_exist/' - ] - with self.assertRaises(exceptions.FileNotFound): - loader.load_tests(path) + # TODO: list/set container with file(s) + # path = [ + # os.path.join(os.getcwd(), 'tests/data_not_exist'), + # 'tests/data_not_exist/' + # ] + # with self.assertRaises(exceptions.FileNotFound): + # loader.load_tests(path) - def test_load_testcases_by_path_layered(self): + def test_load_testcases_with_api_ref(self): path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_layer.yml') - testcases_list = loader.load_tests(path) - self.assertIn("variables", testcases_list[0]["config"]) - self.assertIn("request", testcases_list[0]["config"]) - - # variables in testcase teststep should override api's + tests_mapping = loader.load_tests(path) + project_mapping = tests_mapping["project_mapping"] + testcases_list = tests_mapping["testcases"] + self.assertIn({'device_sn': '${gen_random_string(15)}'}, testcases_list[0]["config"]["variables"]) + self.assertIn("gen_md5", project_mapping["functions"]) + self.assertIn("base_url", testcases_list[0]["config"]) + teststep0 = testcases_list[0]["teststeps"][0] self.assertEqual( - testcases_list[0]["teststeps"][3]["variables"][0]["user_name"], - "user1" + "get token with $user_agent, $app_version", + teststep0["name"] ) - self.assertEqual( - testcases_list[0]["teststeps"][3]["variables"][2]["uid"], - 2000 + self.assertIn("/api/get-token", teststep0["api_def"]["request"]["url"]) + self.assertIn( + {'eq': ['status_code', 200]}, + teststep0["validate"] ) - self.assertIn("request", testcases_list[0]["teststeps"][0]) - self.assertIn("url", testcases_list[0]["teststeps"][0]["request"]) - self.assertIn("validate", testcases_list[0]["teststeps"][0]) + def test_load_testcases_with_testcase_ref(self): + path = os.path.join( + os.getcwd(), 'tests/testsuites/create_users.yml') + tests_mapping = loader.load_tests(path) + project_mapping = tests_mapping["project_mapping"] + testcases_list = tests_mapping["testcases"] + + self.assertEqual( + "create users with uid", + testcases_list[0]["config"]["name"] + ) + self.assertEqual( + {'device_sn': '${gen_random_string(15)}'}, + testcases_list[0]["config"]["variables"][0] + ) + testcase0 = testcases_list[0]["teststeps"][0] + self.assertEqual( + "create user 1000 and check result.", + testcase0["name"] + ) + self.assertEqual( + "create user and check result.", + testcase0["testcase_def"]["config"]["name"] + ) + + testcase1 = testcases_list[0]["teststeps"][1] + self.assertEqual( + "create user 1001 and check result.", + testcase1["name"] + ) + self.assertEqual( + {'uid': 1001}, + testcase1["variables"][0] + ) def test_load_folder_content(self): path = os.path.join(os.getcwd(), "tests", "api") @@ -492,47 +433,16 @@ class TestSuiteLoader(unittest.TestCase): api_definition_mapping = loader.load_api_folder(path) self.assertIn("get_token", api_definition_mapping) self.assertIn("request", api_definition_mapping["get_token"]) - self.assertIn("function_meta", api_definition_mapping["get_token"]) - - def test_load_testcases_folder(self): - path = os.path.join(os.getcwd(), "tests", "suite") - testcases_definition_mapping = loader.load_test_folder(path) - - self.assertIn("setup_and_reset", testcases_definition_mapping) - self.assertIn("create_and_check", testcases_definition_mapping) - self.assertEqual( - testcases_definition_mapping["setup_and_reset"]["config"]["name"], - "setup and reset all." - ) - self.assertEqual( - testcases_definition_mapping["setup_and_reset"]["function_meta"]["func_name"], - "setup_and_reset" - ) - - def test_load_testsuites_folder(self): - path = os.path.join(os.getcwd(), "tests", "testcases") - testsuites_definition_mapping = loader.load_test_folder(path) - - testsute_path = os.path.join(os.getcwd(), "tests", "testcases", "smoketest.yml") - self.assertIn( - testsute_path, - testsuites_definition_mapping - ) - self.assertEqual( - testsuites_definition_mapping[testsute_path]["config"]["name"], - "smoketest" - ) def test_load_project_tests(self): - project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) - self.assertIn("get_token", project_mapping["def-api"]) - self.assertIn("setup_and_reset", project_mapping["def-testcase"]) - self.assertEqual(project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH") + loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + self.assertIn("get_token", self.tests_def_mapping["api"]) + self.assertEqual(self.project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH") def test_load_locust_tests(self): path = os.path.join( os.getcwd(), 'tests/data/demo_locust.yml') locust_tests = loader.load_locust_tests(path) self.assertEqual(len(locust_tests["tests"]), 10) - self.assertEqual(locust_tests["tests"][0][0]["name"], "index") - self.assertEqual(locust_tests["tests"][9][0]["name"], "user-agent") + self.assertEqual(locust_tests["tests"][0]["name"], "index") + self.assertEqual(locust_tests["tests"][9]["name"], "user-agent") diff --git a/tests/test_parser.py b/tests/test_parser.py index 84d9e948..fc4ab158 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -409,7 +409,8 @@ class TestParser(unittest.TestCase): ) def test_parse_parameters_mix(self): - project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + project_mapping = loader.project_mapping parameters = [ {"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]}, @@ -431,7 +432,8 @@ class TestParser(unittest.TestCase): def test_parse_tests(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase.yml') - testcases = loader.load_tests(testcase_file_path) + tests_mapping = loader.load_tests(testcase_file_path) + testcases = tests_mapping["testcases"] self.assertEqual( testcases[0]["config"]["variables"][1]["var_c"], "${sum_two(1, 2)}" @@ -440,17 +442,15 @@ class TestParser(unittest.TestCase): testcases[0]["config"]["variables"][2]["PROJECT_KEY"], "${ENV(PROJECT_KEY)}" ) - - parsed_testcases = parser.parse_tests(testcases) - self.assertEqual(parsed_testcases[0]["config"]["variables"]["var_c"], 3) - self.assertEqual(parsed_testcases[0]["config"]["variables"]["PROJECT_KEY"], "ABCDEFGH") - self.assertEqual(len(parsed_testcases), 2 * 2) - self.assertEqual( - parsed_testcases[0]["config"]["request"]["base_url"], - "http://127.0.0.1:5000" - ) + parser.parse_tests(tests_mapping) + parsed_testcases = tests_mapping["testcases"] self.assertIsInstance(parsed_testcases, list) - self.assertEqual(parsed_testcases[0]["config"]["name"], '12311') + teststep1 = parsed_testcases[0]["teststeps"][0] + self.assertEqual(teststep1["variables"]["var_c"], 3) + self.assertEqual(teststep1["variables"]["PROJECT_KEY"], "ABCDEFGH") + # TODO: parameters + # self.assertEqual(len(parsed_testcases), 2 * 2) + self.assertEqual(parsed_testcases[0]["config"]["name"], '1230') def test_parse_environ(self): os.environ["PROJECT_KEY"] = "ABCDEFGH" @@ -476,3 +476,35 @@ class TestParser(unittest.TestCase): } with self.assertRaises(exceptions.ParamsError): parser.parse_data(content) + + def test_extend_with_api(self): + loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + raw_stepinfo = { + "name": "get token", + "api": "get_token", + } + api_def_dict = loader.load_teststep(raw_stepinfo) + test_block = { + "name": "override block", + "times": 3, + "variables": [ + {"var": 123} + ], + 'request': { + 'url': '/api/get-token', + 'method': 'POST', + 'headers': {'user_agent': '$user_agent', 'device_sn': '$device_sn', 'os_platform': '$os_platform', 'app_version': '$app_version'}, + 'json': {'sign': '${get_sign($user_agent, $device_sn, $os_platform, $app_version)}'} + }, + 'validate': [ + {'eq': ['status_code', 201]}, + {'len_eq': ['content.token', 32]} + ] + } + + extended_block = parser._extend_with_api(test_block, api_def_dict) + self.assertEqual(extended_block["name"], "override block") + self.assertIn({'var': 123}, extended_block["variables"]) + self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, extended_block["validate"]) + self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, extended_block["validate"]) + self.assertEqual(extended_block["times"], 3) diff --git a/tests/test_runner.py b/tests/test_runner.py index a283de4f..ee75e012 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -10,13 +10,16 @@ from tests.base import ApiServerUnittest class TestRunner(ApiServerUnittest): def setUp(self): - project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + loader.load_project_tests(os.path.join(os.getcwd(), "tests")) + project_mapping = loader.project_mapping self.debugtalk_functions = project_mapping["functions"] - config_dict = { - "variables": {}, - "functions": self.debugtalk_functions + + config = { + "name": "XXX", + "base_url": "http://127.0.0.1", + "verify": False } - self.test_runner = runner.Runner(config_dict) + self.test_runner = runner.Runner(config, self.debugtalk_functions) self.reset_all() def reset_all(self): @@ -35,11 +38,8 @@ class TestRunner(ApiServerUnittest): for testcase_file_path in testcase_file_path_list: testcases = loader.load_file(testcase_file_path) - config_dict = { - "variables": {}, - "functions": self.debugtalk_functions - } - test_runner = runner.Runner(config_dict) + config_dict = {} + test_runner = runner.Runner(config_dict, self.debugtalk_functions) test = testcases[0]["test"] test_runner.run_test(test) @@ -81,11 +81,7 @@ class TestRunner(ApiServerUnittest): config_dict = { "name": "basic test with httpbin", - "variables": {}, - "functions": self.debugtalk_functions, - "request": { - "base_url": HTTPBIN_SERVER - }, + "base_url": HTTPBIN_SERVER, "setup_hooks": [ "${sleep_N_secs(0.5)}" "${hook_print(setup)}" @@ -115,7 +111,7 @@ class TestRunner(ApiServerUnittest): {"check": "status_code", "expect": 200} ] } - test_runner = runner.Runner(config_dict) + test_runner = runner.Runner(config_dict, self.debugtalk_functions) end_time = time.time() # check if testcase setup hook executed self.assertGreater(end_time - start_time, 0.5) @@ -130,11 +126,7 @@ class TestRunner(ApiServerUnittest): def test_run_testcase_with_hooks_modify_request(self): config_dict = { "name": "basic test with httpbin", - "variables": {}, - "functions": self.debugtalk_functions, - "request": { - "base_url": HTTPBIN_SERVER - } + "base_url": HTTPBIN_SERVER } test = { "name": "modify request headers", @@ -158,7 +150,7 @@ class TestRunner(ApiServerUnittest): {"check": "content.headers.Os-Platform", "expect": "android"} ] } - test_runner = runner.Runner(config_dict) + test_runner = runner.Runner(config_dict, self.debugtalk_functions) test_runner.run_test(test) def test_run_testcase_with_teardown_hooks_success(self): @@ -183,9 +175,6 @@ class TestRunner(ApiServerUnittest): ], "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] } - config_dict = {} - self.test_runner.init_test(config_dict, "testcase") - start_time = time.time() self.test_runner.run_test(test) end_time = time.time() @@ -214,9 +203,6 @@ class TestRunner(ApiServerUnittest): ], "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] } - config_dict = {} - self.test_runner.init_test(config_dict, "testcase") - start_time = time.time() self.test_runner.run_test(test) end_time = time.time() @@ -226,8 +212,8 @@ class TestRunner(ApiServerUnittest): def test_run_testcase_with_empty_header(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/test_bugfix.yml') - testcases = loader.load_tests(testcase_file_path) - testcase = testcases[0] + tests_mapping = loader.load_tests(testcase_file_path) + testcase = tests_mapping["testcases"][0] config_dict_headers = testcase["config"]["request"]["headers"] test_dict_headers = testcase["teststeps"][0]["request"]["headers"] headers = deep_update_dict( @@ -240,8 +226,6 @@ class TestRunner(ApiServerUnittest): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/test_bugfix.yml') testcases = loader.load_file(testcase_file_path) - config_dict = {} - self.test_runner.init_test(config_dict, "testcase") test = testcases[2]["test"] self.test_runner.run_test(test) diff --git a/tests/test_utils.py b/tests/test_utils.py index 4965fd71..d21d9842 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,8 +2,7 @@ import io import os import shutil -from httprunner import exceptions, loader, utils -from httprunner.compat import OrderedDict +from httprunner import exceptions, loader, parser, utils from tests.base import ApiServerUnittest @@ -206,6 +205,69 @@ class TestUtils(ApiServerUnittest): self.assertIsInstance(ordered_dict, dict) self.assertIn("a", ordered_dict) + def test_extend_validators(self): + def_validators = [ + {'eq': ['v1', 200]}, + {"check": "s2", "expect": 16, "comparator": "len_eq"} + ] + current_validators = [ + {"check": "v1", "expect": 201}, + {'len_eq': ['s3', 12]} + ] + def_validators = [ + parser.parse_validator(validator) + for validator in def_validators + ] + ref_validators = [ + parser.parse_validator(validator) + for validator in current_validators + ] + + extended_validators = utils.extend_validators(def_validators, ref_validators) + self.assertIn( + {"check": "v1", "expect": 201, "comparator": "eq"}, + extended_validators + ) + self.assertIn( + {"check": "s2", "expect": 16, "comparator": "len_eq"}, + extended_validators + ) + self.assertIn( + {"check": "s3", "expect": 12, "comparator": "len_eq"}, + extended_validators + ) + + def test_extend_validators_with_dict(self): + def_validators = [ + {'eq': ["a", {"v": 1}]}, + {'eq': [{"b": 1}, 200]} + ] + current_validators = [ + {'len_eq': ['s3', 12]}, + {'eq': [{"b": 1}, 201]} + ] + def_validators = [ + parser.parse_validator(validator) + for validator in def_validators + ] + ref_validators = [ + parser.parse_validator(validator) + for validator in current_validators + ] + + extended_validators = utils.extend_validators(def_validators, ref_validators) + self.assertEqual(len(extended_validators), 3) + self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'eq'}, extended_validators) + self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'eq'}, extended_validators) + + def test_extend_variables(self): + raw_variables = [{"var1": "val1"}, {"var2": "val2"}] + override_variables = [{"var1": "val111"}, {"var3": "val3"}] + extended_variables_mapping = utils.extend_variables(raw_variables, override_variables) + self.assertEqual(extended_variables_mapping["var1"], "val111") + self.assertEqual(extended_variables_mapping["var2"], "val2") + self.assertEqual(extended_variables_mapping["var3"], "val3") + def test_deepcopy_dict(self): data = { 'a': 1, diff --git a/tests/test_validator.py b/tests/test_validator.py index f83d876f..e3961550 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -12,12 +12,34 @@ class TestValidator(unittest.TestCase): self.assertFalse(validator.is_testcases(data_structure)) data_structure = { - "name": "desc1", - "config": {}, - "api": {}, - "testcases": ["testcase11", "testcase12"] + "project_mapping": { + "PWD": "XXXXX", + "functions": {}, + "env": {} + }, + "testcases": [ + { # testcase data structure + "config": { + "name": "desc1", + "path": "testcase1_path", + "variables": [], # optional + }, + "teststeps": [ + # teststep data structure + { + 'name': 'test step desc1', + 'variables': [], # optional + 'extract': [], # optional + 'validate': [], + 'request': {} + }, + # teststep2 # another teststep dict + ] + }, + # testcase_dict_2 # another testcase dict + ] } - self.assertTrue(data_structure) + self.assertTrue(validator.is_testcases(data_structure)) data_structure = [ { "name": "desc1", diff --git a/tests/suite/create_and_get.yml b/tests/testcases/create_and_check.yml similarity index 54% rename from tests/suite/create_and_get.yml rename to tests/testcases/create_and_check.yml index 425ee621..1736f803 100644 --- a/tests/suite/create_and_get.yml +++ b/tests/testcases/create_and_check.yml @@ -1,35 +1,47 @@ - config: name: "create user and check result." - def: create_and_check($uid, $token) - request: - "base_url": "http://127.0.0.1:5000" - "headers": - "Content-Type": "application/json" - "device_sn": "$device_sn" + id: create_and_check + variables: + - uid: 9001 + - token: XXX + - device_sn: "TESTCASE_CREATE_XXX" + base_url: "http://127.0.0.1:5000" + +- test: + name: setup and reset all (override). + testcase: testcases/setup.yml output: - token - test: name: make sure user $uid does not exist - api: get_user($uid, $token) + api: get_user + variables: + - uid: $uid + - token: $token validate: - eq: ["status_code", 404] - eq: ["content.success", false] - test: name: create user $uid + api: create_user variables: - user_name: "user1" - user_password: "123456" - api: create_user($uid, $user_name, $user_password, $token) + - uid: $uid + - token: $token validate: - eq: ["status_code", 201] - eq: ["content.success", true] - test: name: check if user $uid exists - api: get_user($uid, $token) + api: get_user + variables: + - uid: $uid + - token: $token validate: - eq: ["status_code", 200] - eq: ["content.success", true] diff --git a/tests/suite/setup.yml b/tests/testcases/setup.yml similarity index 57% rename from tests/suite/setup.yml rename to tests/testcases/setup.yml index 126e026b..96522ca9 100644 --- a/tests/suite/setup.yml +++ b/tests/testcases/setup.yml @@ -1,22 +1,19 @@ - config: name: "setup and reset all." - def: setup_and_reset($device_sn) + id: setup_and_reset variables: - user_agent: 'iOS/10.3' - - device_sn: ${gen_random_string(15)} + - device_sn: "TESTCASE_SETUP_XXX" - os_platform: 'ios' - app_version: '2.8.6' - request: - "base_url": "http://127.0.0.1:5000" - "headers": - "Content-Type": "application/json" - "device_sn": "$device_sn" + base_url: "http://127.0.0.1:5000" + verify: False output: - token - test: - name: get token - api: get_token($user_agent, $device_sn, $os_platform, $app_version) + name: get token (setup) + api: get_token variables: - user_agent: 'iOS/10.3' - device_sn: $device_sn @@ -30,4 +27,6 @@ - test: name: reset all users - api: reset_all($token) + api: reset_all + variables: + - token: $token diff --git a/tests/testcases/smoketest.yml b/tests/testcases/smoketest.yml deleted file mode 100644 index d14c61ce..00000000 --- a/tests/testcases/smoketest.yml +++ /dev/null @@ -1,24 +0,0 @@ -- config: - name: smoketest - variables: - - device_sn: ${gen_random_string(15)} - request: - "base_url": "http://127.0.0.1:5000" - "headers": - "Content-Type": "application/json" - "device_sn": "$device_sn" - -- test: - name: setup and reset all. - suite: setup_and_reset($device_sn) - output: - - token - - device_sn - -- test: - name: create user 1000 and check result. - suite: create_and_check(1000, $token) - -- test: - name: create user 1001 and check result. - suite: create_and_check(1001, $token) diff --git a/tests/testsuites/create_users.yml b/tests/testsuites/create_users.yml new file mode 100644 index 00000000..0e344a2e --- /dev/null +++ b/tests/testsuites/create_users.yml @@ -0,0 +1,17 @@ +- config: + name: create users with uid + variables: + - device_sn: ${gen_random_string(15)} + base_url: "http://127.0.0.1:5000" + +- test: + name: create user 1000 and check result. + testcase: testcases/create_and_check.yml + variables: + - uid: 1000 + +- test: + name: create user 1001 and check result. + testcase: testcases/create_and_check.yml + variables: + - uid: 1001 From 1ce95ba3cb9e6123b0f46ca0fbe2e598ab5e48f4 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 22 Nov 2018 19:53:09 +0800 Subject: [PATCH 024/113] build url with base_url --- httprunner/client.py | 16 ++-------------- httprunner/parser.py | 30 +++++++++++++++++++++--------- httprunner/utils.py | 14 ++++++++++++++ 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/httprunner/client.py b/httprunner/client.py index 0f9e4d9a..1a6c1af6 100644 --- a/httprunner/client.py +++ b/httprunner/client.py @@ -1,20 +1,17 @@ # encoding: utf-8 -import re import time import requests import urllib3 from httprunner import logger -from httprunner.exceptions import ParamsError +from httprunner.utils import build_url from requests import Request, Response from requests.exceptions import (InvalidSchema, InvalidURL, MissingSchema, RequestException) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -absolute_http_url_regexp = re.compile(r"^https?://", re.I) - class ApiResponse(Response): @@ -42,15 +39,6 @@ class HttpSession(requests.Session): self.base_url = base_url if base_url else "" self.init_meta_data() - def _build_url(self, path): - """ prepend url with hostname unless it's already an absolute URL """ - if absolute_http_url_regexp.match(path): - return path - elif self.base_url: - return "{}/{}".format(self.base_url.rstrip("/"), path.lstrip("/")) - else: - raise ParamsError("base url missed!") - def init_meta_data(self): """ initialize meta_data, it will store detail data of request and response """ @@ -125,7 +113,7 @@ class HttpSession(requests.Session): self.meta_data["request"]["start_timestamp"] = time.time() # prepend url with hostname unless it's already an absolute URL - url = self._build_url(url) + url = build_url(self.base_url, url) kwargs.setdefault("timeout", 120) response = self._send_request_safe_mode(method, url, **kwargs) diff --git a/httprunner/parser.py b/httprunner/parser.py index 63aac7d8..e764b048 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -609,9 +609,12 @@ def _extend_with_api(teststep_dict, api_def_dict): # TODO: merge & override request teststep_dict["request"] = api_def_dict.pop("request", {}) # base_url - base_url = teststep_dict.pop("base_url", None) - if base_url: - teststep_dict["request"]["url"] = "{}/{}".format(base_url.rstrip("/"), teststep_dict["request"]["url"].lstrip("/")) + if "base_url" in teststep_dict: + base_url = teststep_dict.pop("base_url") + teststep_dict["request"]["url"] = utils.build_url( + base_url, + teststep_dict["request"]["url"] + ) # verify if "verify" in teststep_dict: @@ -744,8 +747,9 @@ def __parse_teststeps(teststeps, config, project_mapping): for teststep in teststeps: - # priority teststep > config - teststep.setdefault("base_url", config_base_url) + # base_url & verify: priority teststep > config + if config_base_url: + teststep.setdefault("base_url", config_base_url) teststep.setdefault("verify", config_verify) if "testcase_def" in teststep: @@ -776,6 +780,10 @@ def __parse_teststeps(teststeps, config, project_mapping): _parse_testcase(teststep, project_mapping) else: + # teststep is API test, has two cases. + # (1) teststep has API reference + # (2) teststep is defined directly + # 1, config => teststeps # override teststep variables teststep["variables"] = utils.extend_variables( @@ -794,14 +802,18 @@ def __parse_teststeps(teststeps, config, project_mapping): pass if "api_def" in teststep: + # case (1) # 2, teststep => api api_def_dict = teststep.pop("api_def") _extend_with_api(teststep, api_def_dict) else: - # base_url - base_url = teststep.pop("base_url", None) - if base_url: - teststep["request"]["url"] = "{}/{}".format(base_url.rstrip("/"), teststep["request"]["url"].lstrip("/")) + # case (2) + if "base_url" in teststep: + base_url = teststep.pop("base_url") + teststep["request"]["url"] = utils.build_url( + base_url, + teststep["request"]["url"] + ) def _parse_testcase(testcase, project_mapping): diff --git a/httprunner/utils.py b/httprunner/utils.py index dfea2059..069b6272 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -6,11 +6,15 @@ import io import itertools import json import os.path +import re import string from datetime import datetime from httprunner import exceptions, logger from httprunner.compat import OrderedDict, basestring, is_py2 +from httprunner.exceptions import ParamsError + +absolute_http_url_regexp = re.compile(r"^https?://", re.I) def set_os_environ(variables_mapping): @@ -48,6 +52,16 @@ def get_os_environ(variable_name): raise exceptions.EnvNotFound(variable_name) +def build_url(base_url, path): + """ prepend url with hostname unless it's already an absolute URL """ + if absolute_http_url_regexp.match(path): + return path + elif base_url: + return "{}/{}".format(base_url.rstrip("/"), path.lstrip("/")) + else: + raise ParamsError("base url missed!") + + def query_json(json_content, query, delimiter='.'): """ Do an xpath-like query with json_content. @param (dict/list/string) json_content From b810ad7e916dd227d54f34b3b64cb6158e9a5870 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 22 Nov 2018 20:43:27 +0800 Subject: [PATCH 025/113] fix environment variable invoke: environment variable maybe loaded in debugtalk.py thus .env file should be loaded before loading debugtalk.py --- httprunner/loader.py | 59 ++++++++++++++++++++++++-------------------- tests/test_loader.py | 12 ++++++--- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index afc79aa0..0f08f279 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -509,40 +509,23 @@ def load_api_folder(api_folder_path): return api_definition_mapping -def load_debugtalk_py(start_path): - """ locate debugtalk.py file and returns PWD and debugtalk.py functions. +def locate_debugtalk_py(start_path): + """ locate debugtalk.py file Args: start_path (str): start locating path, maybe testcase file path or directory path Returns: - tuple: (project_working_directory, debugtalk_functions) + str: debugtalk.py file path, None if not found """ try: # locate debugtalk.py file. debugtalk_path = locate_file(start_path, "debugtalk.py") - - # The folder contains debugtalk.py will be treated as PWD. - project_working_directory = os.path.dirname(debugtalk_path) - - # add PWD to sys.path - sys.path.insert(0, project_working_directory) - - # load debugtalk.py functions - debugtalk_functions = load_debugtalk_functions() - except exceptions.FileNotFound: + debugtalk_path = None - # debugtalk.py not found, use os.getcwd() as PWD. - project_working_directory = os.getcwd() - - # add PWD to sys.path - sys.path.insert(0, project_working_directory) - - debugtalk_functions = {} - - return project_working_directory, debugtalk_functions + return debugtalk_path def load_project_tests(test_path, dot_env_path=None): @@ -557,15 +540,37 @@ def load_project_tests(test_path, dot_env_path=None): dict: project loaded api/testcases definitions, environments and debugtalk.py functions. """ - # locate PWD and load debugtalk.py functions - project_working_directory, debugtalk_functions = load_debugtalk_py(test_path) - project_mapping["PWD"] = project_working_directory - project_mapping["functions"] = debugtalk_functions + # locate debugtalk.py file + debugtalk_path = locate_debugtalk_py(test_path) - # load .env + if debugtalk_path: + # The folder contains debugtalk.py will be treated as PWD. + project_working_directory = os.path.dirname(debugtalk_path) + else: + # debugtalk.py not found, use os.getcwd() as PWD. + project_working_directory = os.getcwd() + + # add PWD to sys.path + sys.path.insert(0, project_working_directory) + + # load .env file + # NOTICE: + # environment variable maybe loaded in debugtalk.py + # thus .env file should be loaded before loading debugtalk.py dot_env_path = dot_env_path or os.path.join(project_working_directory, ".env") project_mapping["env"] = load_dot_env_file(dot_env_path) + if debugtalk_path: + # load debugtalk.py functions + debugtalk_functions = load_debugtalk_functions() + else: + debugtalk_functions = {} + + # locate PWD and load debugtalk.py functions + + project_mapping["PWD"] = project_working_directory + project_mapping["functions"] = debugtalk_functions + # load api tests_def_mapping["api"] = load_api_folder(os.path.join(project_working_directory, "api")) diff --git a/tests/test_loader.py b/tests/test_loader.py index 5630a14a..575802f3 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -208,21 +208,27 @@ class TestModuleLoader(unittest.TestCase): self.assertFalse(is_status_code_200(500)) def test_load_debugtalk_py(self): - project_working_directory, debugtalk_functions = loader.load_debugtalk_py("tests/data/demo_testcase.yml") + loader.load_project_tests("tests/data/demo_testcase.yml") + project_working_directory = loader.project_mapping["PWD"] + debugtalk_functions = loader.project_mapping["functions"] self.assertEqual( project_working_directory, os.path.join(os.getcwd(), "tests") ) self.assertIn("gen_md5", debugtalk_functions) - project_working_directory, debugtalk_functions = loader.load_debugtalk_py("tests/base.py") + loader.load_project_tests("tests/base.py") + project_working_directory = loader.project_mapping["PWD"] + debugtalk_functions = loader.project_mapping["functions"] self.assertEqual( project_working_directory, os.path.join(os.getcwd(), "tests") ) self.assertIn("gen_md5", debugtalk_functions) - project_working_directory, debugtalk_functions = loader.load_debugtalk_py("httprunner/__init__.py") + loader.load_project_tests("httprunner/__init__.py") + project_working_directory = loader.project_mapping["PWD"] + debugtalk_functions = loader.project_mapping["functions"] self.assertEqual( project_working_directory, os.getcwd() From 7d9ffb097f57407d5e0c8a09917958b9ad0bcfde Mon Sep 17 00:00:00 2001 From: debugtalk Date: Thu, 22 Nov 2018 21:48:03 +0800 Subject: [PATCH 026/113] Save loaded tests and parsed tests to JSON file. --- httprunner/api.py | 48 +++++++++++++++++++++------------------------ httprunner/cli.py | 9 ++++++++- httprunner/utils.py | 27 +++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index 97ebde3b..a00ab166 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -134,37 +134,14 @@ class HttpRunner(object): self.summary["details"].append(testcase_summary) - def _run_tests(self, tests_mapping): - """ start to run test with variables mapping. - - Args: - tests_mapping (dict): list of testcase_dict, each testcase is corresponding to a YAML/JSON file - - Returns: - instance: HttpRunner() instance - - """ - self.exception_stage = "parse tests" - parser.parse_tests(tests_mapping) - - self.exception_stage = "add tests to test suite" - test_suite = self._add_tests(tests_mapping) - - self.exception_stage = "run test suite" - results = self._run_suite(test_suite) - - self.exception_stage = "aggregate results" - self._aggregate(results) - - return self - - def run(self, path_or_testcases, dot_env_path=None, mapping=None): + def run(self, path_or_testcases, dot_env_path=None, mapping=None, save_tests=False): """ main interface, run testcases with variables mapping. Args: path_or_testcases (str/list/dict): testcase file/foler path, or valid testcases. dot_env_path (str): specified .env file path. mapping (dict): if mapping is specified, it will override variables in config block. + save_tests (bool): set if save loaded/parsed tests to JSON file. Returns: instance: HttpRunner() instance @@ -177,10 +154,29 @@ class HttpRunner(object): elif validator.is_testcase_path(path_or_testcases): tests_mapping = loader.load_tests(path_or_testcases, dot_env_path) tests_mapping["project_mapping"]["variables"] = mapping or {} + tests_mapping["project_mapping"]["test_path"] = path_or_testcases else: raise exceptions.ParamsError("invalid testcase path or testcases.") - return self._run_tests(tests_mapping) + if save_tests: + utils.dump_tests(tests_mapping, "loaded") + + self.exception_stage = "parse tests" + parser.parse_tests(tests_mapping) + + if save_tests: + utils.dump_tests(tests_mapping, "parsed") + + self.exception_stage = "add tests to test suite" + test_suite = self._add_tests(tests_mapping) + + self.exception_stage = "run test suite" + results = self._run_suite(test_suite) + + self.exception_stage = "aggregate results" + self._aggregate(results) + + return self def gen_html_report(self, html_report_name=None, html_report_template=None): """ generate html report and return report path. diff --git a/httprunner/cli.py b/httprunner/cli.py index d242d456..7c928535 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -45,6 +45,9 @@ def main_hrun(): parser.add_argument( '--failfast', action='store_true', default=False, help="Stop the test run on the first error or failure.") + parser.add_argument( + '--save-tests', action='store_true', default=False, + help="Save loaded tests and parsed tests to JSON file.") parser.add_argument( '--startproject', help="Specify new project name.") @@ -80,7 +83,11 @@ def main_hrun(): for path in args.testcase_paths: try: runner = HttpRunner(failfast=args.failfast) - runner.run(path, dot_env_path=args.dot_env_path) + runner.run( + path, + dot_env_path=args.dot_env_path, + save_tests=args.save_tests + ) except Exception: logger.log_error("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage)) raise diff --git a/httprunner/utils.py b/httprunner/utils.py index 069b6272..9b381fa2 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -583,6 +583,33 @@ def prettify_json_file(file_list): print("success: {}".format(outfile)) +def dump_tests(tests_mapping, tag_name): + """ dump all tests data (except functions) to json file. + + Args: + tests_mapping (dict): data to dump + tag_name (str): tag name, loaded/parsed + + """ + project_mapping = tests_mapping.get("project_mapping", {}) + test_path = project_mapping.get("test_path") or "tests_mapping" + dir_path = project_mapping.get("PWD") or os.getcwd() + file_name, file_suffix = os.path.splitext(os.path.basename(test_path)) + dump_file_path = os.path.join(dir_path, "{}.{}.json".format(file_name, tag_name)) + + tests_to_dump = { + "project_mapping": { + key: project_mapping[key] for key in project_mapping if key != "functions" + }, + "testcases": tests_mapping["testcases"] + } + with open(dump_file_path, 'w', encoding='utf-8') as outfile: + json.dump(tests_to_dump, outfile, indent=4, separators=(',', ': ')) + + msg = "{} file generated successfully: {}".format(tag_name, dump_file_path) + logger.color_print(msg, "BLUE") + + def get_python2_retire_msg(): retire_day = datetime(2020, 1, 1) today = datetime.now() From 09db20e9fcd78df64c4882a6bf0ba1de152f46ef Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 23 Nov 2018 11:19:01 +0800 Subject: [PATCH 027/113] save tests: add debugtalk.py file path --- httprunner/utils.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/httprunner/utils.py b/httprunner/utils.py index 9b381fa2..77515177 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -597,12 +597,19 @@ def dump_tests(tests_mapping, tag_name): file_name, file_suffix = os.path.splitext(os.path.basename(test_path)) dump_file_path = os.path.join(dir_path, "{}.{}.json".format(file_name, tag_name)) - tests_to_dump = { - "project_mapping": { - key: project_mapping[key] for key in project_mapping if key != "functions" - }, - "testcases": tests_mapping["testcases"] - } + tests_to_dump = {} + + for key in project_mapping: + if key != "functions": + tests_to_dump[key] = project_mapping[key] + continue + + if project_mapping["functions"]: + debugtalk_py_path = os.path.join(dir_path, "debugtalk.py") + tests_to_dump["debugtalk.py"] = debugtalk_py_path + + tests_to_dump["testcases"] = tests_mapping["testcases"] + with open(dump_file_path, 'w', encoding='utf-8') as outfile: json.dump(tests_to_dump, outfile, indent=4, separators=(',', ': ')) From 324c0b6f9f486811d0f16749d68d7e99a8d5cf3d Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 23 Nov 2018 11:23:50 +0800 Subject: [PATCH 028/113] move saved tests to logs folder --- httprunner/utils.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/httprunner/utils.py b/httprunner/utils.py index 77515177..dc6f8f14 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -585,6 +585,7 @@ def prettify_json_file(file_list): def dump_tests(tests_mapping, tag_name): """ dump all tests data (except functions) to json file. + the dumped files are located in PWD/logs folder. Args: tests_mapping (dict): data to dump @@ -593,20 +594,27 @@ def dump_tests(tests_mapping, tag_name): """ project_mapping = tests_mapping.get("project_mapping", {}) test_path = project_mapping.get("test_path") or "tests_mapping" - dir_path = project_mapping.get("PWD") or os.getcwd() + pwd_dir_path = project_mapping.get("PWD") or os.getcwd() file_name, file_suffix = os.path.splitext(os.path.basename(test_path)) - dump_file_path = os.path.join(dir_path, "{}.{}.json".format(file_name, tag_name)) - tests_to_dump = {} + logs_dir_path = os.path.join(pwd_dir_path, "logs") + if not os.path.isdir(logs_dir_path): + os.makedirs(logs_dir_path) + + dump_file_path = os.path.join(logs_dir_path, "{}.{}.json".format(file_name, tag_name)) + + tests_to_dump = { + "project_mapping": {} + } for key in project_mapping: if key != "functions": - tests_to_dump[key] = project_mapping[key] + tests_to_dump["project_mapping"][key] = project_mapping[key] continue if project_mapping["functions"]: - debugtalk_py_path = os.path.join(dir_path, "debugtalk.py") - tests_to_dump["debugtalk.py"] = debugtalk_py_path + debugtalk_py_path = os.path.join(pwd_dir_path, "debugtalk.py") + tests_to_dump["project_mapping"]["debugtalk.py"] = debugtalk_py_path tests_to_dump["testcases"] = tests_mapping["testcases"] From 24e6b408e49a7cd5d44a311c0eb4407946af71ea Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 23 Nov 2018 11:41:36 +0800 Subject: [PATCH 029/113] remove empty variables in saved tests --- httprunner/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/httprunner/api.py b/httprunner/api.py index a00ab166..2867bc74 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -153,8 +153,9 @@ class HttpRunner(object): tests_mapping = path_or_testcases elif validator.is_testcase_path(path_or_testcases): tests_mapping = loader.load_tests(path_or_testcases, dot_env_path) - tests_mapping["project_mapping"]["variables"] = mapping or {} tests_mapping["project_mapping"]["test_path"] = path_or_testcases + if "variables" in tests_mapping["project_mapping"]: + tests_mapping["project_mapping"]["variables"] = mapping else: raise exceptions.ParamsError("invalid testcase path or testcases.") From 8f4500566dce58826889d037a18c62caee940777 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 23 Nov 2018 17:14:43 +0800 Subject: [PATCH 030/113] update locusts with HttpRunner 2.0 --- httprunner/api.py | 41 +++++++++++ httprunner/loader.py | 68 ------------------- httprunner/parser.py | 5 -- httprunner/templates/locustfile_template | 34 +++++----- .../demo_simple_locust.yml} | 3 +- tests/test_api.py | 15 +++- tests/test_loader.py | 8 --- 7 files changed, 73 insertions(+), 101 deletions(-) rename tests/{data/demo_locust.yml => locust_tests/demo_simple_locust.yml} (92%) diff --git a/httprunner/api.py b/httprunner/api.py index 2867bc74..3573a0b7 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -199,3 +199,44 @@ class HttpRunner(object): html_report_name, html_report_template ) + + +def prepare_locust_tests(path): + """ prepare locust testcases + + Args: + path (str): testcase file path. + + Returns: + dict: locust tests data + + { + "functions": {}, + "tests": [] + } + + """ + tests_mapping = loader.load_tests(path) + parser.parse_tests(tests_mapping) + + functions = tests_mapping.get("project_mapping", {}).get("functions", {}) + testcase = tests_mapping["testcases"][0] + items = testcase.get("teststeps", []) + + tests = [] + for item in items: + if "config" in item: + # embeded testcase + weight = item["config"].get("weight", 1) + else: + # API test + weight = item.get("weight", 1) + + # implement weight for tests + for _ in range(weight): + tests.append(item) + + return { + "functions": functions, + "tests": tests + } diff --git a/httprunner/loader.py b/httprunner/loader.py index 0f08f279..16593674 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -661,71 +661,3 @@ def load_tests(path, dot_env_path=None): tests_mapping["testcases"] = testcases_list return tests_mapping - - -def load_locust_tests(path, dot_env_path=None): - """ load locust testcases - - Args: - path (str): testcase/testsuite file path. - dot_env_path (str): specified .env file path - - Returns: - dict: locust testcases with weight - { - "config": {...}, - "tests": [ - # weight 3 - [teststep11], - [teststep11], - [teststep11], - # weight 2 - [teststep21, teststep22], - [teststep21, teststep22] - ] - } - - """ - raw_testcase = load_file(path) - load_project_tests(path, dot_env_path) - - config = {} - tests = [] - for item in raw_testcase: - key, test_block = item.popitem() - - if key == "config": - config.update(test_block) - elif key == "test": - teststep = load_teststep(test_block) - weight = test_block.get("weight", 1) - for _ in range(weight): - tests.append(teststep) - - # parse config variables - raw_config_variables = config.get("variables", []) - - config_variables = parser.parse_data( - raw_config_variables, - {}, - project_mapping["functions"] - ) - - # parse config name - config["name"] = parser.parse_data( - config.get("name", ""), - config_variables, - project_mapping["functions"] - ) - - # parse config request - config["request"] = parser.parse_data( - config.get("request", {}), - config_variables, - project_mapping["functions"] - ) - - return { - "config": config, - "tests": tests - } diff --git a/httprunner/parser.py b/httprunner/parser.py index e764b048..59508565 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -860,11 +860,6 @@ def parse_tests(tests_mapping): ] } - variables_mapping (dict): if variables_mapping is specified, it will override variables in config block. - - Returns: - list: parsed testcases list, with config variables/parameters/name/request parsed. - """ project_mapping = tests_mapping.get("project_mapping", {}) diff --git a/httprunner/templates/locustfile_template b/httprunner/templates/locustfile_template index 907086db..32bf497c 100644 --- a/httprunner/templates/locustfile_template +++ b/httprunner/templates/locustfile_template @@ -3,7 +3,7 @@ import random import zmq from httprunner.exceptions import MyBaseError, MyBaseFailure -from httprunner.loader import load_locust_tests +from httprunner.api import prepare_locust_tests from httprunner.runner import Runner from locust import HttpLocust, TaskSet, task from locust.events import request_failure @@ -15,21 +15,20 @@ logging.getLogger('locust.runners').setLevel(logging.INFO) class WebPageTasks(TaskSet): def on_start(self): - self.test_runner = Runner(self.locust.config, self.client) + self.test_runner = Runner(self.locust.config, self.locust.functions, self.client) - @task(weight=1) + @task def test_any(self): - teststeps = random.choice(self.locust.tests) - for teststep in teststeps: - try: - self.test_runner.run_test(teststep) - except (AssertionError, MyBaseError, MyBaseFailure) as ex: - request_failure.fire( - request_type=teststep.get("request", {}).get("method"), - name=teststep.get("request", {}).get("url"), - response_time=0, - exception=ex - ) + test_dict = random.choice(self.locust.tests) + try: + self.test_runner.run_test(test_dict) + except (AssertionError, MyBaseError, MyBaseFailure) as ex: + request_failure.fire( + request_type=test_dict.get("request", {}).get("method"), + name=test_dict.get("request", {}).get("url"), + response_time=0, + exception=ex + ) class WebPageUser(HttpLocust): @@ -38,8 +37,9 @@ class WebPageUser(HttpLocust): max_wait = 30 file_path = "$TESTCASE_FILE" - locust_tests = load_locust_tests(file_path) - config = locust_tests["config"] + locust_tests = prepare_locust_tests(file_path) + functions = locust_tests["functions"] tests = locust_tests["tests"] + config = {} - host = config.get('request', {}).get('base_url', '') + host = config.get('base_url', '') diff --git a/tests/data/demo_locust.yml b/tests/locust_tests/demo_simple_locust.yml similarity index 92% rename from tests/data/demo_locust.yml rename to tests/locust_tests/demo_simple_locust.yml index 8965689c..baab4ae5 100644 --- a/tests/data/demo_locust.yml +++ b/tests/locust_tests/demo_simple_locust.yml @@ -1,7 +1,6 @@ - config: name: basic test with httpbin - request: - base_url: https://httpbin.org/ + base_url: https://httpbin.org/ - test: name: index diff --git a/tests/test_api.py b/tests/test_api.py index 3a1cea67..b6f478da 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ import os import shutil import time +import unittest from httprunner import HttpRunner, api, loader, parser from locust import HttpLocust @@ -564,4 +565,16 @@ class TestApi(ApiServerUnittest): self.assertEqual( results.records[1]["name"], "create user 1001 and check result." - ) \ No newline at end of file + ) + + +class TestLocust(unittest.TestCase): + + def test_prepare_locust_tests(self): + path = os.path.join( + os.getcwd(), 'tests/locust_tests/demo_simple_locust.yml') + locust_tests = api.prepare_locust_tests(path) + self.assertIn("gen_md5", locust_tests["functions"]) + self.assertEqual(len(locust_tests["tests"]), 10) + self.assertEqual(locust_tests["tests"][0]["name"], "index") + self.assertEqual(locust_tests["tests"][9]["name"], "user-agent") diff --git a/tests/test_loader.py b/tests/test_loader.py index 575802f3..7459e39f 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -444,11 +444,3 @@ class TestSuiteLoader(unittest.TestCase): loader.load_project_tests(os.path.join(os.getcwd(), "tests")) self.assertIn("get_token", self.tests_def_mapping["api"]) self.assertEqual(self.project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH") - - def test_load_locust_tests(self): - path = os.path.join( - os.getcwd(), 'tests/data/demo_locust.yml') - locust_tests = loader.load_locust_tests(path) - self.assertEqual(len(locust_tests["tests"]), 10) - self.assertEqual(locust_tests["tests"][0]["name"], "index") - self.assertEqual(locust_tests["tests"][9]["name"], "user-agent") From 18875bc5f19a1f86a4d449ac5cd060f4c4f4a1f0 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 23 Nov 2018 17:56:53 +0800 Subject: [PATCH 031/113] rename teststeps to tests --- httprunner/api.py | 20 ++--- httprunner/context.py | 22 ++--- httprunner/loader.py | 44 +++++----- httprunner/parser.py | 180 ++++++++++++++++++++-------------------- httprunner/runner.py | 92 ++++++++++---------- httprunner/validator.py | 16 ++-- tests/test_api.py | 86 +++++++++---------- tests/test_context.py | 14 ++-- tests/test_loader.py | 50 +++++------ tests/test_parser.py | 10 +-- tests/test_runner.py | 2 +- tests/test_validator.py | 6 +- 12 files changed, 271 insertions(+), 271 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index 3573a0b7..bb9d3090 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -38,12 +38,12 @@ class HttpRunner(object): unittest.TestSuite() """ - def _add_teststep(test_runner, teststep_dict): - """ add teststep to testcase. + def _add_test(test_runner, test_dict): + """ add test to testcase. """ def test(self): try: - test_runner.run_test(teststep_dict) + test_runner.run_test(test_dict) except exceptions.MyBaseFailure as ex: self.fail(str(ex)) finally: @@ -53,7 +53,7 @@ class HttpRunner(object): test_runner.http_client_session.init_meta_data() # TODO: refactor - test.__doc__ = teststep_dict.get("name") or teststep_dict.get("config", {}).get("name") + test.__doc__ = test_dict.get("name") or test_dict.get("config", {}).get("name") return test test_suite = unittest.TestSuite() @@ -64,18 +64,18 @@ class HttpRunner(object): test_runner = runner.Runner(config, functions, self.http_client_session) TestSequense = type('TestSequense', (unittest.TestCase,), {}) - teststeps = testcase.get("teststeps", []) - for index, teststep_dict in enumerate(teststeps): - for times_index in range(int(teststep_dict.get("times", 1))): + tests = testcase.get("tests", []) + for index, test_dict in enumerate(tests): + for times_index in range(int(test_dict.get("times", 1))): # suppose one testcase should not have more than 9999 steps, # and one step should not run more than 999 times. test_method_name = 'test_{:04}_{:03}'.format(index, times_index) - test_method = _add_teststep(test_runner, teststep_dict) + test_method = _add_test(test_runner, test_dict) setattr(TestSequense, test_method_name, test_method) loaded_testcase = self.test_loader.loadTestsFromTestCase(TestSequense) setattr(loaded_testcase, "config", config) - setattr(loaded_testcase, "teststeps", teststeps) + setattr(loaded_testcase, "tests", tests) setattr(loaded_testcase, "runner", test_runner) test_suite.addTest(loaded_testcase) @@ -221,7 +221,7 @@ def prepare_locust_tests(path): functions = tests_mapping.get("project_mapping", {}).get("functions", {}) testcase = tests_mapping["testcases"][0] - items = testcase.get("teststeps", []) + items = testcase.get("tests", []) tests = [] for item in items: diff --git a/httprunner/context.py b/httprunner/context.py index c4d98546..b7d42b03 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -17,11 +17,11 @@ class SessionContext(object): def __init__(self, functions, variables=None): self.session_variables_mapping = utils.ensure_mapping_format(variables or {}) self.FUNCTIONS_MAPPING = functions - self.teststep_variables_mapping = {} - self.init_teststep_variables() + self.test_variables_mapping = {} + self.init_test_variables() - def init_teststep_variables(self, variables_mapping=None): - """ init teststep variables, called when each teststep(api) starts. + def init_test_variables(self, variables_mapping=None): + """ init test variables, called when each test(api) starts. variables_mapping will be evaluated first. Args: @@ -38,14 +38,14 @@ class SessionContext(object): variables_mapping = utils.ensure_mapping_format(variables_mapping) for variable_name, variable_value in variables_mapping.items(): variable_value = self.eval_content(variable_value) - self.update_teststep_variables(variable_name, variable_value) + self.update_test_variables(variable_name, variable_value) - self.teststep_variables_mapping.update(self.session_variables_mapping) + self.test_variables_mapping.update(self.session_variables_mapping) - def update_teststep_variables(self, variable_name, variable_value): - """ update teststep variables, these variables are only valid in the current teststep. + def update_test_variables(self, variable_name, variable_value): + """ update test variables, these variables are only valid in the current test. """ - self.teststep_variables_mapping[variable_name] = variable_value + self.test_variables_mapping[variable_name] = variable_value def update_seesion_variables(self, variables_mapping): """ update session with extracted variables mapping. @@ -53,7 +53,7 @@ class SessionContext(object): """ variables_mapping = utils.ensure_mapping_format(variables_mapping) self.session_variables_mapping.update(variables_mapping) - self.teststep_variables_mapping.update(self.session_variables_mapping) + self.test_variables_mapping.update(self.session_variables_mapping) def eval_content(self, content): """ evaluate content recursively, take effect on each variable and function in content. @@ -61,7 +61,7 @@ class SessionContext(object): """ return parser.parse_data( content, - self.teststep_variables_mapping, + self.test_variables_mapping, self.FUNCTIONS_MAPPING ) diff --git a/httprunner/loader.py b/httprunner/loader.py index 16593674..5d04ceae 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -279,11 +279,11 @@ tests_def_mapping = { "testcases": {} } -def load_teststep(raw_stepinfo): - """ load teststep with api/testcase/proc references +def load_test(raw_testinfo): + """ load test with api/testcase/proc references Args: - raw_stepinfo (dict): teststep data, maybe in 3 formats. + raw_testinfo (dict): test data, maybe in 3 formats. # api reference { "name": "add product to cart", @@ -308,24 +308,24 @@ def load_teststep(raw_stepinfo): } Returns: - list: loaded teststeps list + list: loaded tests list Args: - raw_stepinfo (dict): teststep info + raw_testinfo (dict): test info """ # reference api - if "api" in raw_stepinfo: - api_name = raw_stepinfo["api"] - raw_stepinfo["api_def"] = _get_api_definition(api_name) + if "api" in raw_testinfo: + api_name = raw_testinfo["api"] + raw_testinfo["api_def"] = _get_api_definition(api_name) # TODO: reference proc functions - elif "func" in raw_stepinfo: + elif "func" in raw_testinfo: pass # reference testcase - elif "testcase" in raw_stepinfo: - testcase_path = raw_stepinfo["testcase"] + elif "testcase" in raw_testinfo: + testcase_path = raw_testinfo["testcase"] if testcase_path not in tests_def_mapping["testcases"]: testcase_path = os.path.join( @@ -337,13 +337,13 @@ def load_teststep(raw_stepinfo): else: testcase_dict = tests_def_mapping[testcase_path] - raw_stepinfo["testcase_def"] = testcase_dict + raw_testinfo["testcase_def"] = testcase_dict # define directly else: pass - return raw_stepinfo + return raw_testinfo def load_testcase(raw_testcase): @@ -360,7 +360,7 @@ def load_testcase(raw_testcase): "request": {} } }, - # teststeps part + # tests part { "test": {...} }, @@ -374,12 +374,12 @@ def load_testcase(raw_testcase): { "name": "XYZ", "config": {}, - "teststeps": [teststep11, teststep12] + "tests": [test11, test12] } """ config = {} - teststeps = [] + tests = [] for item in raw_testcase: # TODO: add json schema validation @@ -394,7 +394,7 @@ def load_testcase(raw_testcase): config.update(test_block) elif key == "test": - teststeps.append(load_teststep(test_block)) + tests.append(load_test(test_block)) else: logger.log_warning( @@ -403,7 +403,7 @@ def load_testcase(raw_testcase): return { "config": config, - "teststeps": teststeps + "tests": tests } @@ -601,16 +601,16 @@ def load_tests(path, dot_env_path=None): "path": "testcase1_path", "variables": [], # optional }, - "teststeps": [ - # teststep data structure + "tests": [ + # test data structure { - 'name': 'test step desc1', + 'name': 'test desc1', 'variables': [], # optional 'extract': [], # optional 'validate': [], 'request': {} }, - teststep2 # another teststep dict + test_dict_2 # another test dict ] }, testcase_dict_2 # another testcase dict diff --git a/httprunner/parser.py b/httprunner/parser.py index 59508565..201bc74c 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -542,15 +542,15 @@ def parse_data(content, variables_mapping=None, functions_mapping=None): return content -def _extend_with_api(teststep_dict, api_def_dict): - """ extend teststep with api definition, teststep will merge and override api definition. +def _extend_with_api(test_dict, api_def_dict): + """ extend test with api definition, test will merge and override api definition. Args: - teststep_dict (dict): teststep block + test_dict (dict): test block api_def_dict (dict): api definition Returns: - dict: extended teststep dict. + dict: extended test dict. Examples: >>> api_def_dict = { @@ -558,12 +558,12 @@ def _extend_with_api(teststep_dict, api_def_dict): "request": {...}, "validate": [{'eq': ['status_code', 200]}] } - >>> teststep_dict = { + >>> test_dict = { "name": "get token 2", "extract": [{"token": "content.token"}], "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] } - >>> _extend_with_api(teststep_dict, api_def_dict) + >>> _extend_with_api(test_dict, api_def_dict) { "name": "get token 2", "request": {...}, @@ -574,18 +574,18 @@ def _extend_with_api(teststep_dict, api_def_dict): """ # override name api_def_name = api_def_dict.pop("name", "") - teststep_dict["name"] = teststep_dict.get("name") or api_def_name + test_dict["name"] = test_dict.get("name") or api_def_name # override variables def_variables = api_def_dict.pop("variables", []) - teststep_dict["variables"] = utils.extend_variables( + test_dict["variables"] = utils.extend_variables( def_variables, - teststep_dict.get("variables", {}) + test_dict.get("variables", {}) ) # merge & override validators TODO: relocate def_raw_validators = api_def_dict.pop("validate", []) - ref_raw_validators = teststep_dict.get("validate", []) + ref_raw_validators = test_dict.get("validate", []) def_validators = [ parse_validator(validator) for validator in def_raw_validators @@ -594,86 +594,86 @@ def _extend_with_api(teststep_dict, api_def_dict): parse_validator(validator) for validator in ref_raw_validators ] - teststep_dict["validate"] = utils.extend_validators( + test_dict["validate"] = utils.extend_validators( def_validators, ref_validators ) # merge & override extractors def_extrators = api_def_dict.pop("extract", []) - teststep_dict["extract"] = utils.extend_variables( + test_dict["extract"] = utils.extend_variables( def_extrators, - teststep_dict.get("extract", []) + test_dict.get("extract", []) ) # TODO: merge & override request - teststep_dict["request"] = api_def_dict.pop("request", {}) + test_dict["request"] = api_def_dict.pop("request", {}) # base_url - if "base_url" in teststep_dict: - base_url = teststep_dict.pop("base_url") - teststep_dict["request"]["url"] = utils.build_url( + if "base_url" in test_dict: + base_url = test_dict.pop("base_url") + test_dict["request"]["url"] = utils.build_url( base_url, - teststep_dict["request"]["url"] + test_dict["request"]["url"] ) # verify - if "verify" in teststep_dict: - verify = teststep_dict.pop("verify") + if "verify" in test_dict: + verify = test_dict.pop("verify") elif "verify" in api_def_dict: verify = api_def_dict.pop("verify") else: verify = True - teststep_dict["request"]["verify"] = verify + test_dict["request"]["verify"] = verify # merge & override setup_hooks def_setup_hooks = api_def_dict.pop("setup_hooks", []) - ref_setup_hooks = teststep_dict.get("setup_hooks", []) + ref_setup_hooks = test_dict.get("setup_hooks", []) extended_setup_hooks = list(set(def_setup_hooks + ref_setup_hooks)) if extended_setup_hooks: - teststep_dict["setup_hooks"] = extended_setup_hooks + test_dict["setup_hooks"] = extended_setup_hooks # merge & override teardown_hooks def_teardown_hooks = api_def_dict.pop("teardown_hooks", []) - ref_teardown_hooks = teststep_dict.get("teardown_hooks", []) + ref_teardown_hooks = test_dict.get("teardown_hooks", []) extended_teardown_hooks = list(set(def_teardown_hooks + ref_teardown_hooks)) if extended_teardown_hooks: - teststep_dict["teardown_hooks"] = extended_teardown_hooks + test_dict["teardown_hooks"] = extended_teardown_hooks # TODO: extend with other api definition items, e.g. times - teststep_dict.update(api_def_dict) + test_dict.update(api_def_dict) - return teststep_dict + return test_dict -def _extend_with_testcase(teststep_dict, testcase_def_dict): - """ extend teststep with testcase definition - teststep will merge and override testcase config definition. +def _extend_with_testcase(test_dict, testcase_def_dict): + """ extend test with testcase definition + test will merge and override testcase config definition. Args: - teststep_dict (dict): teststep block + test_dict (dict): test block testcase_def_dict (dict): testcase definition Returns: - dict: extended teststep dict. + dict: extended test dict. """ # override testcase config variables testcase_def_dict["config"].setdefault("variables", {}) testcase_def_variables = utils.ensure_mapping_format(testcase_def_dict["config"].get("variables", {})) - testcase_def_variables.update(teststep_dict.pop("variables", {})) + testcase_def_variables.update(test_dict.pop("variables", {})) testcase_def_dict["config"]["variables"] = testcase_def_variables # override base_url, verify - # priority: testcase config > testsuite teststep - teststep_base_url = teststep_dict.pop("base_url", None) - teststep_verify = teststep_dict.pop("verify", True) - testcase_def_dict["config"].setdefault("base_url", teststep_base_url) - testcase_def_dict["config"].setdefault("verify", teststep_verify) + # priority: testcase config > testsuite tests + test_base_url = test_dict.pop("base_url", None) + test_verify = test_dict.pop("verify", True) + testcase_def_dict["config"].setdefault("base_url", test_base_url) + testcase_def_dict["config"].setdefault("verify", test_verify) # override testcase config name, output, etc. - testcase_def_dict["config"].update(teststep_dict) + testcase_def_dict["config"].update(test_dict) - teststep_dict.clear() - teststep_dict.update(testcase_def_dict) + test_dict.clear() + test_dict.update(testcase_def_dict) def __parse_config(config, project_mapping): @@ -722,22 +722,22 @@ def __parse_config(config, project_mapping): ) -def __parse_teststeps(teststeps, config, project_mapping): - """ override teststeps with testcase config variables, base_url and verify. - teststep maybe nested testcase. +def __parse_tests(tests, config, project_mapping): + """ override tests with testcase config variables, base_url and verify. + test maybe nested testcase. variables priority: - testsuite config > testsuite teststep > testcase config > testcase teststep > api + testsuite config > testsuite test > testcase config > testcase test > api base_url/verify priority: - testcase teststep > testcase config > testsuite teststep > testsuite config > api + testcase test > testcase config > testsuite test > testsuite config > api Args: - teststeps (list): + tests (list): config (dict): Returns: - list: overrided teststeps + list: overrided tests """ config_variables = config.pop("variables", {}) @@ -745,80 +745,80 @@ def __parse_teststeps(teststeps, config, project_mapping): config_verify = config.pop("verify", True) functions = project_mapping.get("functions", {}) - for teststep in teststeps: + for test_dict in tests: - # base_url & verify: priority teststep > config + # base_url & verify: priority test_dict > config if config_base_url: - teststep.setdefault("base_url", config_base_url) - teststep.setdefault("verify", config_verify) + test_dict.setdefault("base_url", config_base_url) + test_dict.setdefault("verify", config_verify) - if "testcase_def" in teststep: - # teststep is nested testcase + if "testcase_def" in test_dict: + # test_dict is nested testcase - # 1, testsuite config => testsuite teststeps - # override teststep variables - teststep["variables"] = utils.extend_variables( - teststep.pop("variables", {}), + # 1, testsuite config => testsuite tests + # override test_dict variables + test_dict["variables"] = utils.extend_variables( + test_dict.pop("variables", {}), config_variables ) - # parse teststep name + # parse test_dict name try: - teststep["name"] = parse_data( - teststep.pop("name", ""), - teststep["variables"], + test_dict["name"] = parse_data( + test_dict.pop("name", ""), + test_dict["variables"], functions ) except exceptions.VariableNotFound: pass - # 2, testsuite teststep => testcase config - testcase_def = teststep.pop("testcase_def") - _extend_with_testcase(teststep, testcase_def) + # 2, testsuite test_dict => testcase config + testcase_def = test_dict.pop("testcase_def") + _extend_with_testcase(test_dict, testcase_def) - # 3, testcase config => testcase teststep - _parse_testcase(teststep, project_mapping) + # 3, testcase config => testcase test_dict + _parse_testcase(test_dict, project_mapping) else: - # teststep is API test, has two cases. - # (1) teststep has API reference - # (2) teststep is defined directly + # test_dict is API test, has two cases. + # (1) test_dict has API reference + # (2) test_dict is defined directly - # 1, config => teststeps - # override teststep variables - teststep["variables"] = utils.extend_variables( - teststep.pop("variables", {}), + # 1, config => tests + # override test_dict variables + test_dict["variables"] = utils.extend_variables( + test_dict.pop("variables", {}), config_variables ) - # parse teststep name + # parse test_dict name try: - teststep["name"] = parse_data( - teststep.pop("name", ""), - teststep["variables"], + test_dict["name"] = parse_data( + test_dict.pop("name", ""), + test_dict["variables"], functions ) except exceptions.VariableNotFound: pass - if "api_def" in teststep: + if "api_def" in test_dict: # case (1) - # 2, teststep => api - api_def_dict = teststep.pop("api_def") - _extend_with_api(teststep, api_def_dict) + # 2, test_dict => api + api_def_dict = test_dict.pop("api_def") + _extend_with_api(test_dict, api_def_dict) else: # case (2) - if "base_url" in teststep: - base_url = teststep.pop("base_url") - teststep["request"]["url"] = utils.build_url( + if "base_url" in test_dict: + base_url = test_dict.pop("base_url") + test_dict["request"]["url"] = utils.build_url( base_url, - teststep["request"]["url"] + test_dict["request"]["url"] ) def _parse_testcase(testcase, project_mapping): __parse_config(testcase["config"], project_mapping) - __parse_teststeps(testcase["teststeps"], testcase["config"], project_mapping) + __parse_tests(testcase["tests"], testcase["config"], project_mapping) def parse_tests(tests_mapping): @@ -841,8 +841,8 @@ def parse_tests(tests_mapping): "path": "testcase1_path", "variables": [], # optional, priority 2 }, - "teststeps": [ - # teststep data structure + "tests": [ + # test data structure { 'name': 'test step desc1', 'variables': [], # optional, priority 3 @@ -853,7 +853,7 @@ def parse_tests(tests_mapping): 'request': {}, } }, - teststep2 # another teststep dict + test_dict_2 # another test dict ] }, testcase_dict_2 # another testcase dict diff --git a/httprunner/runner.py b/httprunner/runner.py index e38aa7aa..5a1d9a51 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -19,15 +19,15 @@ class Runner(object): } >>> runner = Runner(config, functions) - >>> teststep = { - "name": "teststep description", + >>> test_dict = { + "name": "test description", "variables": [], # optional "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "GET" } } - >>> runner.run_test(teststep) + >>> runner.run_test(test_dict) """ @@ -68,32 +68,32 @@ class Runner(object): if self.testcase_teardown_hooks: self.do_hook_actions(self.testcase_teardown_hooks) - def _handle_skip_feature(self, teststep_dict): - """ handle skip feature for teststep + def _handle_skip_feature(self, test_dict): + """ handle skip feature for test - skip: skip current test unconditionally - skipIf: skip current test if condition is true - skipUnless: skip current test unless condition is true Args: - teststep_dict (dict): teststep info + test_dict (dict): test info Raises: - SkipTest: skip teststep + SkipTest: skip test """ # TODO: move skip to initialize skip_reason = None - if "skip" in teststep_dict: - skip_reason = teststep_dict["skip"] + if "skip" in test_dict: + skip_reason = test_dict["skip"] - elif "skipIf" in teststep_dict: - skip_if_condition = teststep_dict["skipIf"] + elif "skipIf" in test_dict: + skip_if_condition = test_dict["skipIf"] if self.session_context.eval_content(skip_if_condition): skip_reason = "{} evaluate to True".format(skip_if_condition) - elif "skipUnless" in teststep_dict: - skip_unless_condition = teststep_dict["skipUnless"] + elif "skipUnless" in test_dict: + skip_unless_condition = test_dict["skipUnless"] if not self.session_context.eval_content(skip_unless_condition): skip_reason = "{} evaluate to False".format(skip_unless_condition) @@ -106,11 +106,11 @@ class Runner(object): # TODO: check hook function if valid self.session_context.eval_content(action) - def _run_teststep(self, teststep_dict): + def _run_test(self, test_dict): """ run single teststep. Args: - teststep_dict (dict): teststep info + test_dict (dict): teststep info { "name": "teststep description", "skip": "skip this test unconditionally", @@ -139,28 +139,28 @@ class Runner(object): """ # check skip - self._handle_skip_feature(teststep_dict) + self._handle_skip_feature(test_dict) # prepare - teststep_dict = utils.lower_test_dict_keys(teststep_dict) - teststep_variables = teststep_dict.get("variables", {}) - self.session_context.init_teststep_variables(teststep_variables) + test_dict = utils.lower_test_dict_keys(test_dict) + test_variables = test_dict.get("variables", {}) + self.session_context.init_test_variables(test_variables) - # parse teststep request - raw_request = teststep_dict.get('request', {}) - parsed_teststep_request = self.session_context.eval_content(raw_request) - self.session_context.update_teststep_variables("request", parsed_teststep_request) + # parse test request + raw_request = test_dict.get('request', {}) + parsed_test_request = self.session_context.eval_content(raw_request) + self.session_context.update_test_variables("request", parsed_test_request) # setup hooks - setup_hooks = teststep_dict.get("setup_hooks", []) + setup_hooks = test_dict.get("setup_hooks", []) setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}") self.do_hook_actions(setup_hooks) try: - url = parsed_teststep_request.pop('url') - method = parsed_teststep_request.pop('method') - parsed_teststep_request.setdefault("verify", self.verify) - group_name = parsed_teststep_request.pop("group", None) + url = parsed_test_request.pop('url') + method = parsed_test_request.pop('method') + parsed_test_request.setdefault("verify", self.verify) + group_name = parsed_test_request.pop("group", None) except KeyError: raise exceptions.ParamsError("URL or METHOD missed!") @@ -173,38 +173,38 @@ class Runner(object): raise exceptions.ParamsError(err_msg) logger.log_info("{method} {url}".format(method=method, url=url)) - logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_teststep_request)) + logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request)) # request resp = self.http_client_session.request( method, url, name=group_name, - **parsed_teststep_request + **parsed_test_request ) resp_obj = response.ResponseObject(resp) # teardown hooks - teardown_hooks = teststep_dict.get("teardown_hooks", []) + teardown_hooks = test_dict.get("teardown_hooks", []) if teardown_hooks: logger.log_info("start to run teardown hooks") - self.session_context.update_teststep_variables("response", resp_obj) + self.session_context.update_test_variables("response", resp_obj) self.do_hook_actions(teardown_hooks) # extract - extractors = teststep_dict.get("extract", []) + extractors = test_dict.get("extract", []) extracted_variables_mapping = resp_obj.extract_response(extractors) self.session_context.update_seesion_variables(extracted_variables_mapping) # validate - validators = teststep_dict.get("validate", []) + validators = test_dict.get("validate", []) try: self.evaluated_validators = self.session_context.validate(validators, resp_obj) except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure): # log request err_req_msg = "request: \n" - err_req_msg += "headers: {}\n".format(parsed_teststep_request.pop("headers", {})) - for k, v in parsed_teststep_request.items(): + err_req_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {})) + for k, v in parsed_test_request.items(): err_req_msg += "{}: {}\n".format(k, repr(v)) logger.log_error(err_req_msg) @@ -223,18 +223,18 @@ class Runner(object): config = testcase_dict.get("config", {}) test_runner = Runner(config, self.functions, self.http_client_session) - teststeps = testcase_dict.get("teststeps", []) - for index, teststep_dict in enumerate(teststeps): - test_runner.run_test(teststep_dict) + tests = testcase_dict.get("tests", []) + for index, test_dict in enumerate(tests): + test_runner.run_test(test_dict) self.session_context.update_seesion_variables(test_runner.extract_sessions()) - def run_test(self, teststep_dict): + def run_test(self, test_dict): """ run single teststep of testcase. - teststep_dict may be in 3 types. + test_dict may be in 3 types. Args: - teststep_dict (dict): + test_dict (dict): # teststep { @@ -249,7 +249,7 @@ class Runner(object): # embeded testcase { "config": {...}, - "teststeps": [ + "tests": [ {...}, {...} ] @@ -262,11 +262,11 @@ class Runner(object): } """ - if "config" in teststep_dict: - self._run_testcase(teststep_dict) + if "config" in test_dict: + self._run_testcase(test_dict) else: # api - self._run_teststep(teststep_dict) + self._run_test(test_dict) def extract_sessions(self): """ diff --git a/httprunner/validator.py b/httprunner/validator.py index 70b44eeb..fcccc7c6 100644 --- a/httprunner/validator.py +++ b/httprunner/validator.py @@ -19,9 +19,9 @@ def is_testcase(data_structure): "variables": [], # optional "request": {} # optional }, - "teststeps": [ - teststep1, - { # teststep2 + "tests": [ + test_dict1, + { # test_dict2 'name': 'test step desc2', 'variables': [], # optional 'extract': [], # optional @@ -40,10 +40,10 @@ def is_testcase(data_structure): if not isinstance(data_structure, dict): return False - if "teststeps" not in data_structure: + if "tests" not in data_structure: return False - if not isinstance(data_structure["teststeps"], list): + if not isinstance(data_structure["tests"], list): return False return True @@ -67,8 +67,8 @@ def is_testcases(data_structure): "path": "testcase1_path", "variables": [], # optional }, - "teststeps": [ - # teststep data structure + "tests": [ + # test data structure { 'name': 'test step desc1', 'variables': [], # optional @@ -76,7 +76,7 @@ def is_testcases(data_structure): 'validate': [], 'request': {} }, - teststep2 # another teststep dict + test_dict_2 # another test dict ] }, testcase_dict_2 # another testcase dict diff --git a/tests/test_api.py b/tests/test_api.py index b6f478da..a1cb4649 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -28,7 +28,7 @@ class TestHttpRunner(ApiServerUnittest): }, 'variables': [] }, - 'teststeps': [ + 'tests': [ { 'name': '/api/get-token', 'request': { @@ -119,7 +119,7 @@ class TestHttpRunner(ApiServerUnittest): }, 'variables': [] }, - "teststeps": [ + "tests": [ { "name": "post data", "request": { @@ -182,7 +182,7 @@ class TestHttpRunner(ApiServerUnittest): testcases = [ { "config": {"name": "test teardown hooks"}, - "teststeps": [ + "tests": [ { "name": "test teardown hooks", "request": { @@ -222,7 +222,7 @@ class TestHttpRunner(ApiServerUnittest): "config": { "name": "test teardown hooks" }, - "teststeps": [ + "tests": [ { "name": "test teardown hooks", "request": { @@ -256,7 +256,7 @@ class TestHttpRunner(ApiServerUnittest): "config": { "name": "test teardown hooks" }, - "teststeps": [ + "tests": [ { "name": "test teardown hooks", "request": { @@ -368,28 +368,28 @@ class TestHttpRunner(ApiServerUnittest): test_suite = runner._add_tests(tests_mapping) self.assertEqual( - test_suite._tests[0].teststeps[0]['name'], + test_suite._tests[0].tests[0]['name'], 'get token with iOS/10.1 and test1' ) # TODO: add parameterize # self.assertEqual( - # test_suite._tests[1].teststeps[0]['name'], + # test_suite._tests[1].tests[0]['name'], # 'get token with iOS/10.1 and test2' # ) # self.assertEqual( - # test_suite._tests[2].teststeps[0]['name'], + # test_suite._tests[2].tests[0]['name'], # 'get token with iOS/10.2 and test1' # ) # self.assertEqual( - # test_suite._tests[3].teststeps[0]['name'], + # test_suite._tests[3].tests[0]['name'], # 'get token with iOS/10.2 and test2' # ) # self.assertEqual( - # test_suite._tests[4].teststeps[0]['name'], + # test_suite._tests[4].tests[0]['name'], # 'get token with iOS/10.3 and test1' # ) # self.assertEqual( - # test_suite._tests[5].teststeps[0]['name'], + # test_suite._tests[5].tests[0]['name'], # 'get token with iOS/10.3 and test2' # ) @@ -419,13 +419,13 @@ class TestApi(ApiServerUnittest): self.assertEqual(testcase_config["name"], "setup and reset all.") self.assertIn("path", testcase_config) - testcase_teststeps = testcases[0]["teststeps"] - self.assertEqual(len(testcase_teststeps), 2) - self.assertIn("api", testcase_teststeps[0]) - self.assertEqual(testcase_teststeps[0]["name"], "get token (setup)") - self.assertIsInstance(testcase_teststeps[0]["variables"], list) - self.assertIn("api_def", testcase_teststeps[0]) - self.assertEqual(testcase_teststeps[0]["api_def"]["request"]["url"], "/api/get-token") + testcase_tests = testcases[0]["tests"] + self.assertEqual(len(testcase_tests), 2) + self.assertIn("api", testcase_tests[0]) + self.assertEqual(testcase_tests[0]["name"], "get token (setup)") + self.assertIsInstance(testcase_tests[0]["variables"], list) + self.assertIn("api_def", testcase_tests[0]) + self.assertEqual(testcase_tests[0]["api_def"]["request"]["url"], "/api/get-token") def test_testcase_parser(self): testcase_path = "tests/testcases/setup.yml" @@ -437,13 +437,13 @@ class TestApi(ApiServerUnittest): self.assertEqual(len(parsed_testcases), 1) self.assertNotIn("variables", parsed_testcases[0]["config"]) - self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2) + self.assertEqual(len(parsed_testcases[0]["tests"]), 2) - teststep1 = parsed_testcases[0]["teststeps"][0] - self.assertEqual(teststep1["name"], "get token (setup)") - self.assertNotIn("api_def", teststep1) - self.assertEqual(teststep1["variables"]["device_sn"], "TESTCASE_SETUP_XXX") - self.assertEqual(teststep1["request"]["url"], "http://127.0.0.1:5000/api/get-token") + test_dict1 = parsed_testcases[0]["tests"][0] + self.assertEqual(test_dict1["name"], "get token (setup)") + self.assertNotIn("api_def", test_dict1) + self.assertEqual(test_dict1["variables"]["device_sn"], "TESTCASE_SETUP_XXX") + self.assertEqual(test_dict1["request"]["url"], "http://127.0.0.1:5000/api/get-token") def test_testcase_add_tests(self): testcase_path = "tests/testcases/setup.yml" @@ -454,10 +454,10 @@ class TestApi(ApiServerUnittest): test_suite = runner._add_tests(tests_mapping) self.assertEqual(len(test_suite._tests), 1) - teststeps = test_suite._tests[0].teststeps - self.assertEqual(teststeps[0]["name"], "get token (setup)") - self.assertEqual(teststeps[0]["variables"]["device_sn"], "TESTCASE_SETUP_XXX") - self.assertIn("api", teststeps[0]) + tests = test_suite._tests[0].tests + self.assertEqual(tests[0]["name"], "get token (setup)") + self.assertEqual(tests[0]["variables"]["device_sn"], "TESTCASE_SETUP_XXX") + self.assertIn("api", tests[0]) def test_testcase_simple_run_suite(self): testcase_path = "tests/testcases/setup.yml" @@ -504,14 +504,14 @@ class TestApi(ApiServerUnittest): self.assertEqual(testcase_config["name"], "create users with uid") self.assertIn("path", testcase_config) - testcase_teststeps = testcases[0]["teststeps"] - self.assertEqual(len(testcase_teststeps), 2) - self.assertIn("testcase_def", testcase_teststeps[0]) - self.assertEqual(testcase_teststeps[0]["name"], "create user 1000 and check result.") - self.assertIsInstance(testcase_teststeps[0]["testcase_def"], dict) - self.assertEqual(testcase_teststeps[0]["testcase_def"]["config"]["name"], "create user and check result.") - self.assertEqual(len(testcase_teststeps[0]["testcase_def"]["teststeps"]), 4) - self.assertEqual(testcase_teststeps[0]["testcase_def"]["teststeps"][0]["name"], "setup and reset all (override).") + testcase_tests = testcases[0]["tests"] + self.assertEqual(len(testcase_tests), 2) + self.assertIn("testcase_def", testcase_tests[0]) + self.assertEqual(testcase_tests[0]["name"], "create user 1000 and check result.") + self.assertIsInstance(testcase_tests[0]["testcase_def"], dict) + self.assertEqual(testcase_tests[0]["testcase_def"]["config"]["name"], "create user and check result.") + self.assertEqual(len(testcase_tests[0]["testcase_def"]["tests"]), 4) + self.assertEqual(testcase_tests[0]["testcase_def"]["tests"][0]["name"], "setup and reset all (override).") def test_testsuite_parser(self): testcase_path = "tests/testsuites/create_users.yml" @@ -521,17 +521,17 @@ class TestApi(ApiServerUnittest): parsed_testcases = tests_mapping["testcases"] self.assertEqual(len(parsed_testcases), 1) - self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2) + self.assertEqual(len(parsed_testcases[0]["tests"]), 2) - testcase1 = parsed_testcases[0]["teststeps"][0] + testcase1 = parsed_testcases[0]["tests"][0] self.assertEqual(testcase1["config"]["name"], "create user 1000 and check result.") self.assertNotIn("testcase_def", testcase1) - self.assertEqual(len(testcase1["teststeps"]), 4) + self.assertEqual(len(testcase1["tests"]), 4) self.assertEqual( - testcase1["teststeps"][0]["teststeps"][0]["request"]["url"], + testcase1["tests"][0]["tests"][0]["request"]["url"], "http://127.0.0.1:5000/api/get-token" ) - self.assertEqual(len(testcase1["teststeps"][0]["teststeps"][0]["variables"]["device_sn"]), 15) + self.assertEqual(len(testcase1["tests"][0]["tests"][0]["variables"]["device_sn"]), 15) def test_testsuite_add_tests(self): testcase_path = "tests/testsuites/create_users.yml" @@ -542,8 +542,8 @@ class TestApi(ApiServerUnittest): test_suite = runner._add_tests(tests_mapping) self.assertEqual(len(test_suite._tests), 1) - teststeps = test_suite._tests[0].teststeps - self.assertEqual(teststeps[0]["config"]["name"], "create user 1000 and check result.") + tests = test_suite._tests[0].tests + self.assertEqual(tests[0]["config"]["name"], "create user 1000 and check result.") def test_testsuite_run_suite(self): testcase_path = "tests/testsuites/create_users.yml" diff --git a/tests/test_context.py b/tests/test_context.py index 809b4195..d61f186a 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -22,9 +22,9 @@ class TestContext(ApiServerUnittest): context_functions = self.context.FUNCTIONS_MAPPING self.assertIn("gen_md5", context_functions) - def test_init_teststep_variables(self): + def test_init_test_variables(self): self.assertEqual( - self.context.teststep_variables_mapping, + self.context.test_variables_mapping, {'SECRET_KEY': 'DebugTalk'} ) @@ -63,7 +63,7 @@ class TestContext(ApiServerUnittest): {"data": '{"name": "user", "password": "123456"}'}, {"authorization": "${gen_md5($TOKEN, $data, $random)}"} ] - self.context.init_teststep_variables(variables) + self.context.init_test_variables(variables) request = { "url": "http://127.0.0.1:5000/api/users/1000", @@ -110,7 +110,7 @@ class TestContext(ApiServerUnittest): {"resp_status_code": 200}, {"resp_body_success": True} ] - self.context.init_teststep_variables(variables) + self.context.init_test_variables(variables) with self.assertRaises(exceptions.ValidationFailure): self.context.validate(validators, resp_obj) @@ -125,7 +125,7 @@ class TestContext(ApiServerUnittest): {"resp_status_code": 201}, {"resp_body_success": True} ] - self.context.init_teststep_variables(variables) + self.context.init_test_variables(variables) self.context.validate(validators, resp_obj) def test_validate_exception(self): @@ -139,7 +139,7 @@ class TestContext(ApiServerUnittest): {"check": "$resp_status_code", "comparator": "eq", "expect": 201} ] variables = [] - self.context.init_teststep_variables(variables) + self.context.init_test_variables(variables) with self.assertRaises(exceptions.VariableNotFound): self.context.validate(validators, resp_obj) @@ -148,7 +148,7 @@ class TestContext(ApiServerUnittest): variables = [ {"resp_status_code": 200} ] - self.context.init_teststep_variables(variables) + self.context.init_test_variables(variables) with self.assertRaises(exceptions.ValidationFailure): self.context.validate(validators, resp_obj) diff --git a/tests/test_loader.py b/tests/test_loader.py index 7459e39f..64d160c1 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -264,8 +264,8 @@ class TestSuiteLoader(unittest.TestCase): cls.project_mapping = loader.project_mapping cls.tests_def_mapping = loader.tests_def_mapping - def test_load_teststep_testcase(self): - raw_teststep = { + def test_load_test_testcase(self): + raw_test = { "name": "setup and reset all (override).", "testcase": "testcases/setup.yml", "variables": [ @@ -273,24 +273,24 @@ class TestSuiteLoader(unittest.TestCase): ], "output": ["token", "device_sn"] } - testcase = loader.load_teststep(raw_teststep) + testcase = loader.load_test(raw_test) self.assertEqual( "setup and reset all (override).", testcase["name"] ) - teststeps = testcase["testcase_def"]["teststeps"] - self.assertEqual(len(teststeps), 2) - self.assertEqual(teststeps[0]["name"], "get token (setup)") - self.assertEqual(teststeps[1]["name"], "reset all users") + tests = testcase["testcase_def"]["tests"] + self.assertEqual(len(tests), 2) + self.assertEqual(tests[0]["name"], "get token (setup)") + self.assertEqual(tests[1]["name"], "reset all users") def test_load_testcase(self): raw_testcase = loader.load_file("tests/testsuites/create_users.yml") testcase = loader.load_testcase(raw_testcase) self.assertEqual(testcase["config"]["name"], "create users with uid") self.assertIn("device_sn", testcase["config"]["variables"][0]) - self.assertEqual(len(testcase["teststeps"]), 2) - self.assertEqual(testcase["teststeps"][0]["name"], "create user 1000 and check result.") - self.assertEqual(testcase["teststeps"][0]["testcase_def"]["config"]["name"], "create user and check result.") + self.assertEqual(len(testcase["tests"]), 2) + self.assertEqual(testcase["tests"][0]["name"], "create user 1000 and check result.") + self.assertEqual(testcase["tests"][0]["testcase_def"]["config"]["name"], "create user and check result.") def test_load_testcases_by_path_files(self): # absolute file path @@ -300,7 +300,7 @@ class TestSuiteLoader(unittest.TestCase): project_mapping = tests_mapping["project_mapping"] testcases_list = tests_mapping["testcases"] self.assertEqual(len(testcases_list), 1) - self.assertEqual(len(testcases_list[0]["teststeps"]), 3) + self.assertEqual(len(testcases_list[0]["tests"]), 3) self.assertIn("get_sign", project_mapping["functions"]) # relative file path @@ -309,7 +309,7 @@ class TestSuiteLoader(unittest.TestCase): project_mapping = tests_mapping["project_mapping"] testcases_list = tests_mapping["testcases"] self.assertEqual(len(testcases_list), 1) - self.assertEqual(len(testcases_list[0]["teststeps"]), 3) + self.assertEqual(len(testcases_list[0]["tests"]), 3) self.assertIn("get_sign", project_mapping["functions"]) # TODO: list/set container with file(s) @@ -319,17 +319,17 @@ class TestSuiteLoader(unittest.TestCase): # ] # testcases_list = loader.load_tests(path) # self.assertEqual(len(testcases_list), 2) - # self.assertEqual(len(testcases_list[0]["teststeps"]), 3) - # self.assertEqual(len(testcases_list[1]["teststeps"]), 3) + # self.assertEqual(len(testcases_list[0]["tests"]), 3) + # self.assertEqual(len(testcases_list[1]["tests"]), 3) # testcases_list.extend(testcases_list) # self.assertEqual(len(testcases_list), 4) # for testcase in testcases_list: - # for teststep in testcase["teststeps"]: - # self.assertIn('name', teststep) - # self.assertIn('request', teststep) - # self.assertIn('url', teststep['request']) - # self.assertIn('method', teststep['request']) + # for test_dict in testcase["tests"]: + # self.assertIn('name', test_dict) + # self.assertIn('request', test_dict) + # self.assertIn('url', test_dict['request']) + # self.assertIn('method', test_dict['request']) def test_load_testcases_by_path_folder(self): # absolute folder path @@ -381,15 +381,15 @@ class TestSuiteLoader(unittest.TestCase): self.assertIn({'device_sn': '${gen_random_string(15)}'}, testcases_list[0]["config"]["variables"]) self.assertIn("gen_md5", project_mapping["functions"]) self.assertIn("base_url", testcases_list[0]["config"]) - teststep0 = testcases_list[0]["teststeps"][0] + test_dict0 = testcases_list[0]["tests"][0] self.assertEqual( "get token with $user_agent, $app_version", - teststep0["name"] + test_dict0["name"] ) - self.assertIn("/api/get-token", teststep0["api_def"]["request"]["url"]) + self.assertIn("/api/get-token", test_dict0["api_def"]["request"]["url"]) self.assertIn( {'eq': ['status_code', 200]}, - teststep0["validate"] + test_dict0["validate"] ) def test_load_testcases_with_testcase_ref(self): @@ -407,7 +407,7 @@ class TestSuiteLoader(unittest.TestCase): {'device_sn': '${gen_random_string(15)}'}, testcases_list[0]["config"]["variables"][0] ) - testcase0 = testcases_list[0]["teststeps"][0] + testcase0 = testcases_list[0]["tests"][0] self.assertEqual( "create user 1000 and check result.", testcase0["name"] @@ -417,7 +417,7 @@ class TestSuiteLoader(unittest.TestCase): testcase0["testcase_def"]["config"]["name"] ) - testcase1 = testcases_list[0]["teststeps"][1] + testcase1 = testcases_list[0]["tests"][1] self.assertEqual( "create user 1001 and check result.", testcase1["name"] diff --git a/tests/test_parser.py b/tests/test_parser.py index fc4ab158..5bf8353d 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -445,9 +445,9 @@ class TestParser(unittest.TestCase): parser.parse_tests(tests_mapping) parsed_testcases = tests_mapping["testcases"] self.assertIsInstance(parsed_testcases, list) - teststep1 = parsed_testcases[0]["teststeps"][0] - self.assertEqual(teststep1["variables"]["var_c"], 3) - self.assertEqual(teststep1["variables"]["PROJECT_KEY"], "ABCDEFGH") + test_dict1 = parsed_testcases[0]["tests"][0] + self.assertEqual(test_dict1["variables"]["var_c"], 3) + self.assertEqual(test_dict1["variables"]["PROJECT_KEY"], "ABCDEFGH") # TODO: parameters # self.assertEqual(len(parsed_testcases), 2 * 2) self.assertEqual(parsed_testcases[0]["config"]["name"], '1230') @@ -479,11 +479,11 @@ class TestParser(unittest.TestCase): def test_extend_with_api(self): loader.load_project_tests(os.path.join(os.getcwd(), "tests")) - raw_stepinfo = { + raw_testinfo = { "name": "get token", "api": "get_token", } - api_def_dict = loader.load_teststep(raw_stepinfo) + api_def_dict = loader.load_test(raw_testinfo) test_block = { "name": "override block", "times": 3, diff --git a/tests/test_runner.py b/tests/test_runner.py index ee75e012..624d076b 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -215,7 +215,7 @@ class TestRunner(ApiServerUnittest): tests_mapping = loader.load_tests(testcase_file_path) testcase = tests_mapping["testcases"][0] config_dict_headers = testcase["config"]["request"]["headers"] - test_dict_headers = testcase["teststeps"][0]["request"]["headers"] + test_dict_headers = testcase["tests"][0]["request"]["headers"] headers = deep_update_dict( config_dict_headers, test_dict_headers diff --git a/tests/test_validator.py b/tests/test_validator.py index e3961550..1797b0cb 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -24,8 +24,8 @@ class TestValidator(unittest.TestCase): "path": "testcase1_path", "variables": [], # optional }, - "teststeps": [ - # teststep data structure + "tests": [ + # test data structure { 'name': 'test step desc1', 'variables': [], # optional @@ -33,7 +33,7 @@ class TestValidator(unittest.TestCase): 'validate': [], 'request': {} }, - # teststep2 # another teststep dict + # test_dict2 # another test dict ] }, # testcase_dict_2 # another testcase dict From cfd1d84189bc5209b1b0455310f04ae901509fdf Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 23 Nov 2018 18:27:29 +0800 Subject: [PATCH 032/113] record response content: 1, only record json if response content is in JSON foramt; 2, do not record bytes content; 3, omit text charactors greater than 1000. --- httprunner/client.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/httprunner/client.py b/httprunner/client.py index 1a6c1af6..c62d12aa 100644 --- a/httprunner/client.py +++ b/httprunner/client.py @@ -56,7 +56,6 @@ class HttpSession(requests.Session): "response_time_ms": "N/A", "elapsed_ms": "N/A", "encoding": None, - "content": None, "content_type": "" } } @@ -139,14 +138,20 @@ class HttpSession(requests.Session): self.meta_data["response"]["headers"] = dict(response.headers) self.meta_data["response"]["cookies"] = response.cookies or {} self.meta_data["response"]["encoding"] = response.encoding - self.meta_data["response"]["content"] = response.content - self.meta_data["response"]["text"] = response.text self.meta_data["response"]["content_type"] = response.headers.get("Content-Type", "") try: + # try to only record json data self.meta_data["response"]["json"] = response.json() except ValueError: - self.meta_data["response"]["json"] = None + # only record at most 1000 text charactors + resp_text = response.text + resp_text_length = len(resp_text) + if resp_text_length > 1000: + resp_text = resp_text[0:1000] \ + + " ... OMITTED {} CHARACTORS ...".format(resp_text_length-1000) + + self.meta_data["response"]["text"] = resp_text # get the length of the content, but if the argument stream is set to True, we take # the size from the content-length header, in order to not trigger fetching of the body From 23cf17cb3974b9bd77958b7681c44ea82d3ec524 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Fri, 23 Nov 2018 22:52:23 +0800 Subject: [PATCH 033/113] convert docstring to Google style --- httprunner/loader.py | 26 +++++++++------- httprunner/parser.py | 35 +++++++++++++--------- httprunner/response.py | 67 ++++++++++++++++++++++++++++-------------- httprunner/utils.py | 61 +++++++++++++++++++++++++------------- 4 files changed, 123 insertions(+), 66 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index 5d04ceae..d03cbde8 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -58,21 +58,27 @@ def load_json_file(json_file): def load_csv_file(csv_file): """ load csv file and check file content format - @param - csv_file: csv file path - e.g. csv file content: - username,password - test1,111111 - test2,222222 - test3,333333 - @return - list of parameter, each parameter is in dict format - e.g. + + Args: + csv_file (str): csv file path, csv file content is like below: + + Returns: + list: list of parameters, each parameter is in dict format + + Examples: + >>> cat csv_file + username,password + test1,111111 + test2,222222 + test3,333333 + + >>> load_csv_file(csv_file) [ {'username': 'test1', 'password': '111111'}, {'username': 'test2', 'password': '222222'}, {'username': 'test3', 'password': '333333'} ] + """ csv_content_list = [] diff --git a/httprunner/parser.py b/httprunner/parser.py index 201bc74c..c3756472 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -150,20 +150,27 @@ def parse_function(content): def parse_validator(validator): - """ parse validator, validator maybe in two format - @param (dict) validator - format1: this is kept for compatiblity with the previous versions. - {"check": "status_code", "comparator": "eq", "expect": 201} - {"check": "$resp_body_success", "comparator": "eq", "expect": True} - format2: recommended new version - {'eq': ['status_code', 201]} - {'eq': ['$resp_body_success', True]} - @return (dict) validator info - { - "check": "status_code", - "expect": 201, - "comparator": "eq" - } + """ parse validator + + Args: + validator (dict): validator maybe in two formats: + + format1: this is kept for compatiblity with the previous versions. + {"check": "status_code", "comparator": "eq", "expect": 201} + {"check": "$resp_body_success", "comparator": "eq", "expect": True} + format2: recommended new version + {'eq': ['status_code', 201]} + {'eq': ['$resp_body_success', True]} + + Returns + dict: validator info + + { + "check": "status_code", + "expect": 201, + "comparator": "eq" + } + """ if not isinstance(validator, dict): raise exceptions.ParamsError("invalid validator: {}".format(validator)) diff --git a/httprunner/response.py b/httprunner/response.py index 07f3c868..46b3af5d 100644 --- a/httprunner/response.py +++ b/httprunner/response.py @@ -15,7 +15,10 @@ class ResponseObject(object): def __init__(self, resp_obj): """ initialize with a requests.Response object - @param (requests.Response instance) resp_obj + + Args: + resp_obj (instance): requests.Response instance + """ self.resp_obj = resp_obj @@ -36,11 +39,22 @@ class ResponseObject(object): def _extract_field_with_regex(self, field): """ extract field from response content with regex. requests.Response body could be json or html text. - @param (str) field should only be regex string that matched r".*\(.*\).*" - e.g. - self.text: "LB123abcRB789" - field: "LB[\d]*(.*)RB[\d]*" - return: abc + + Args: + field (str): regex string that matched r".*\(.*\).*" + + Returns: + str: matched content. + + Raises: + exceptions.ExtractFailure: If no content matched with regex. + + Examples: + >>> # self.text: "LB123abcRB789" + >>> filed = "LB[\d]*(.*)RB[\d]*" + >>> _extract_field_with_regex(field) + abc + """ matched = re.search(field, self.text) if not matched: @@ -53,14 +67,17 @@ class ResponseObject(object): def _extract_field_with_delimiter(self, field): """ response content could be json or html text. - @param (str) field should be string joined by delimiter. - e.g. - "status_code" - "headers" - "cookies" - "content" - "headers.content-type" - "content.person.name.first_name" + + Args: + field (str): string joined by delimiter. + e.g. + "status_code" + "headers" + "cookies" + "content" + "headers.content-type" + "content.person.name.first_name" + """ # string.split(sep=None, maxsplit=-1) -> list of strings # e.g. "content.person.name" => ["content", "person.name"] @@ -207,14 +224,20 @@ class ResponseObject(object): def extract_response(self, extractors): """ extract value from requests.Response and store in OrderedDict. - @param (list) extractors - [ - {"resp_status_code": "status_code"}, - {"resp_headers_content_type": "headers.content-type"}, - {"resp_content": "content"}, - {"resp_content_person_first_name": "content.person.name.first_name"} - ] - @return (OrderDict) variable binds ordered dict + + Args: + extractors (list): + + [ + {"resp_status_code": "status_code"}, + {"resp_headers_content_type": "headers.content-type"}, + {"resp_content": "content"}, + {"resp_content_person_first_name": "content.person.name.first_name"} + ] + + Returns: + OrderDict: variable binds ordered dict + """ if not extractors: return {} diff --git a/httprunner/utils.py b/httprunner/utils.py index dc6f8f14..20f6f2eb 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -64,8 +64,17 @@ def build_url(base_url, path): def query_json(json_content, query, delimiter='.'): """ Do an xpath-like query with json_content. - @param (dict/list/string) json_content - json_content = { + + Args: + json_content (dict/list/string): content to be queried. + query (str): query string. + delimiter (str): delimiter symbol. + + Returns: + str: queried result. + + Examples: + >>> json_content = { "ids": [1, 2, 3, 4], "person": { "name": { @@ -76,11 +85,16 @@ def query_json(json_content, query, delimiter='.'): "cities": ["Guangzhou", "Shenzhen"] } } - @param (str) query - "person.name.first_name" => "Leo" - "person.name.first_name.0" => "L" - "person.cities.0" => "Guangzhou" - @return queried result + >>> + >>> query_json(json_content, "person.name.first_name") + >>> Leo + >>> + >>> query_json(json_content, "person.name.first_name.0") + >>> L + >>> + >>> query_json(json_content, "person.cities.0") + >>> Guangzhou + """ raise_flag = False response_body = u"response body: {}\n".format(json_content) @@ -505,21 +519,28 @@ def create_scaffold(project_name): def gen_cartesian_product(*args): """ generate cartesian product for lists - @param - (list) args - [{"a": 1}, {"a": 2}], + + Args: + args (list of list): lists to be generated with cartesian product + + Returns: + list: cartesian product in list + + Examples: + + >>> arg1 = [{"a": 1}, {"a": 2}] + >>> arg2 = [{"x": 111, "y": 112}, {"x": 121, "y": 122}] + >>> args = [arg1, arg2] + >>> gen_cartesian_product(*args) + >>> # same as below + >>> gen_cartesian_product(arg1, arg2) [ - {"x": 111, "y": 112}, - {"x": 121, "y": 122} + {'a': 1, 'x': 111, 'y': 112}, + {'a': 1, 'x': 121, 'y': 122}, + {'a': 2, 'x': 111, 'y': 112}, + {'a': 2, 'x': 121, 'y': 122} ] - @return - cartesian product in list - [ - {'a': 1, 'x': 111, 'y': 112}, - {'a': 1, 'x': 121, 'y': 122}, - {'a': 2, 'x': 111, 'y': 112}, - {'a': 2, 'x': 121, 'y': 122} - ] + """ if not args: return [] From 5d3dc6ad1c03e37da8157d5f7424cc7730ecf6cd Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 24 Nov 2018 00:00:25 +0800 Subject: [PATCH 034/113] bugfix: test_variables_mapping should be initialized before each test --- httprunner/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httprunner/context.py b/httprunner/context.py index b7d42b03..66bb3af8 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -17,7 +17,6 @@ class SessionContext(object): def __init__(self, functions, variables=None): self.session_variables_mapping = utils.ensure_mapping_format(variables or {}) self.FUNCTIONS_MAPPING = functions - self.test_variables_mapping = {} self.init_test_variables() def init_test_variables(self, variables_mapping=None): @@ -34,6 +33,7 @@ class SessionContext(object): ] """ + self.test_variables_mapping = {} variables_mapping = variables_mapping or {} variables_mapping = utils.ensure_mapping_format(variables_mapping) for variable_name, variable_value in variables_mapping.items(): From 4fbaa71e7e72c43aa8f96bc99d4c69dda8e06487 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 24 Nov 2018 00:23:57 +0800 Subject: [PATCH 035/113] implement hook assignment: save returned value from hook function to teststep scope variable --- httprunner/runner.py | 40 ++++++++++++++++++++++++++++++++-------- tests/test_runner.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/httprunner/runner.py b/httprunner/runner.py index 5a1d9a51..db622bd9 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -62,11 +62,11 @@ class Runner(object): self.session_context = SessionContext(self.functions) if testcase_setup_hooks: - self.do_hook_actions(testcase_setup_hooks) + self.do_hook_actions(testcase_setup_hooks, "setup") def __del__(self): if self.testcase_teardown_hooks: - self.do_hook_actions(self.testcase_teardown_hooks) + self.do_hook_actions(self.testcase_teardown_hooks, "teardown") def _handle_skip_feature(self, test_dict): """ handle skip feature for test @@ -100,11 +100,35 @@ class Runner(object): if skip_reason: raise SkipTest(skip_reason) - def do_hook_actions(self, actions): + def do_hook_actions(self, actions, hook_type): + """ call hook actions. + + Args: + actions (str/dict): actions maybe in two format. + + format1: only call hook functions. + ${func()} + format2: assignment, the value returned by hook function will be assigned to variable. + {"var": "${func()}"} + + hook_type (enum): setup/teardown + + """ + logger.log_info("call {} hook actions.".format(hook_type)) for action in actions: - logger.log_debug("call hook: {}".format(action)) - # TODO: check hook function if valid - self.session_context.eval_content(action) + + if isinstance(action, dict) and len(action) == 1: + # {"var": "${func()}"} + var_name, hook_content = list(action.items())[0] + logger.log_debug("assignment with hook: {} = {}".format(var_name, hook_content)) + self.session_context.update_test_variables( + var_name, + self.session_context.eval_content(hook_content) + ) + else: + logger.log_debug("call hook function: {}".format(action)) + # TODO: check hook function if valid + self.session_context.eval_content(action) def _run_test(self, test_dict): """ run single teststep. @@ -154,7 +178,7 @@ class Runner(object): # setup hooks setup_hooks = test_dict.get("setup_hooks", []) setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}") - self.do_hook_actions(setup_hooks) + self.do_hook_actions(setup_hooks, "setup") try: url = parsed_test_request.pop('url') @@ -189,7 +213,7 @@ class Runner(object): if teardown_hooks: logger.log_info("start to run teardown hooks") self.session_context.update_test_variables("response", resp_obj) - self.do_hook_actions(teardown_hooks) + self.do_hook_actions(teardown_hooks, "teardown") # extract extractors = test_dict.get("extract", []) diff --git a/tests/test_runner.py b/tests/test_runner.py index 624d076b..ce87b5e2 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -123,6 +123,35 @@ class TestRunner(ApiServerUnittest): # testcase teardown hook has not been executed now self.assertLess(end_time - start_time, 1) + def test_run_testcase_with_hooks_assignment(self): + config_dict = { + "name": "basic test with httpbin", + "base_url": HTTPBIN_SERVER + } + test = { + "name": "modify request headers", + "request": { + "url": "/anything", + "method": "POST", + "headers": { + "user_agent": "iOS/10.3", + "os_platform": "ios" + }, + "data": "a=1&b=2" + }, + "setup_hooks": [ + {"total": "${sum_two(1, 5)}"} + ], + "validate": [ + {"check": "status_code", "expect": 200} + ] + } + test_runner = runner.Runner(config_dict, self.debugtalk_functions) + test_runner.run_test(test) + test_variables_mapping = test_runner.session_context.test_variables_mapping + self.assertEqual(test_variables_mapping["total"], 6) + self.assertEqual(test_variables_mapping["request"]["data"], "a=1&b=2") + def test_run_testcase_with_hooks_modify_request(self): config_dict = { "name": "basic test with httpbin", From 3ea4789c7bd3a6b9db363155de2d0d1d60b67d4e Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 24 Nov 2018 00:49:13 +0800 Subject: [PATCH 036/113] fix commit f1be0972dae59389c21b0371580c3101f67bc808 --- httprunner/context.py | 4 ++-- tests/testcases/create_and_check.yml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/httprunner/context.py b/httprunner/context.py index 66bb3af8..aa06663e 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -34,14 +34,14 @@ class SessionContext(object): """ self.test_variables_mapping = {} + self.test_variables_mapping.update(self.session_variables_mapping) + variables_mapping = variables_mapping or {} variables_mapping = utils.ensure_mapping_format(variables_mapping) for variable_name, variable_value in variables_mapping.items(): variable_value = self.eval_content(variable_value) self.update_test_variables(variable_name, variable_value) - self.test_variables_mapping.update(self.session_variables_mapping) - def update_test_variables(self, variable_name, variable_value): """ update test variables, these variables are only valid in the current test. """ diff --git a/tests/testcases/create_and_check.yml b/tests/testcases/create_and_check.yml index 1736f803..959a9da3 100644 --- a/tests/testcases/create_and_check.yml +++ b/tests/testcases/create_and_check.yml @@ -4,7 +4,6 @@ id: create_and_check variables: - uid: 9001 - - token: XXX - device_sn: "TESTCASE_CREATE_XXX" base_url: "http://127.0.0.1:5000" From 61df6cdaae8c1aeb06524d5c9001c17e58d13a8b Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 24 Nov 2018 10:28:00 +0800 Subject: [PATCH 037/113] fix #363: clear meta data first to ensure independence for each test --- httprunner/api.py | 14 ++++++++------ httprunner/report.py | 5 +---- httprunner/runner.py | 19 ++++++++++++++++++- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index bb9d3090..6f8770eb 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -47,13 +47,15 @@ class HttpRunner(object): except exceptions.MyBaseFailure as ex: self.fail(str(ex)) finally: - if hasattr(test_runner.http_client_session, "meta_data"): - self.meta_data = test_runner.http_client_session.meta_data - self.meta_data["validators"] = test_runner.evaluated_validators - test_runner.http_client_session.init_meta_data() + self.meta_data = test_runner.get_test_data() + + if "config" in test_dict: + # run nested testcase + test.__doc__ = test_dict["config"].get("name") + else: + # run api test + test.__doc__ = test_dict.get("name") - # TODO: refactor - test.__doc__ = test_dict.get("name") or test_dict.get("config", {}).get("name") return test test_suite = unittest.TestSuite() diff --git a/httprunner/report.py b/httprunner/report.py index b3b3300d..0da55305 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -186,11 +186,8 @@ class HtmlTestResult(unittest.TextTestResult): 'name': test.shortDescription(), 'status': status, 'attachment': attachment, - "meta_data": {} + "meta_data": test.meta_data } - if hasattr(test, "meta_data"): - data["meta_data"] = test.meta_data - self.records.append(data) def startTestRun(self): diff --git a/httprunner/runner.py b/httprunner/runner.py index db622bd9..7dd05360 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -68,6 +68,19 @@ class Runner(object): if self.testcase_teardown_hooks: self.do_hook_actions(self.testcase_teardown_hooks, "teardown") + def clear_test_data(self): + """ clear request and response data + """ + self.evaluated_validators = [] + self.http_client_session.init_meta_data() + + def get_test_data(self): + """ get request/response data and validate results + """ + meta_data = self.http_client_session.meta_data + meta_data["validators"] = self.evaluated_validators + return meta_data + def _handle_skip_feature(self, test_dict): """ handle skip feature for test - skip: skip current test unconditionally @@ -162,6 +175,9 @@ class Runner(object): exceptions.ExtractFailure """ + # clear meta data first to ensure independence for each test + self.clear_test_data() + # check skip self._handle_skip_feature(test_dict) @@ -270,7 +286,7 @@ class Runner(object): } } - # embeded testcase + # nested testcase { "config": {...}, "tests": [ @@ -287,6 +303,7 @@ class Runner(object): """ if "config" in test_dict: + # nested testcase self._run_testcase(test_dict) else: # api From d26baf8a4f3865a53dc9ee59ff8fea32e983a797 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 25 Nov 2018 20:48:16 +0800 Subject: [PATCH 038/113] remove unused code --- httprunner/api.py | 8 ++------ httprunner/parser.py | 7 ------- httprunner/report.py | 22 +++++++++++----------- 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index 6f8770eb..2d17af46 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -16,14 +16,11 @@ class HttpRunner(object): kwargs (dict): key-value arguments used to initialize TextTestRunner. Commonly used arguments: - resultclass (class): HtmlTestResult or TextTestResult failfast (bool): False/True, stop the test run on the first error or failure. - http_client_session (instance): requests.Session(), or locust.client.Session() instance. """ self.exception_stage = "initialize HttpRunner()" - self.http_client_session = kwargs.pop("http_client_session", None) - kwargs.setdefault("resultclass", report.HtmlTestResult) + kwargs["resultclass"] = report.HtmlTestResult self.unittest_runner = unittest.TextTestRunner(**kwargs) self.test_loader = unittest.TestLoader() self.summary = None @@ -63,7 +60,7 @@ class HttpRunner(object): for testcase in tests_mapping["testcases"]: config = testcase.get("config", {}) - test_runner = runner.Runner(config, functions, self.http_client_session) + test_runner = runner.Runner(config, functions) TestSequense = type('TestSequense', (unittest.TestCase,), {}) tests = testcase.get("tests", []) @@ -125,7 +122,6 @@ class HttpRunner(object): self.summary["success"] &= testcase_summary["success"] testcase_summary["name"] = testcase.config.get("name") - testcase_summary["base_url"] = testcase.config.get("request", {}).get("base_url", "") in_out = utils.get_testcase_io(testcase) utils.print_io(in_out) diff --git a/httprunner/parser.py b/httprunner/parser.py index c3756472..7853778f 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -870,13 +870,6 @@ def parse_tests(tests_mapping): """ project_mapping = tests_mapping.get("project_mapping", {}) - env_mapping = project_mapping.get("env", {}) - # set OS environment variables - utils.set_os_environ(env_mapping) - for testcase in tests_mapping["testcases"]: testcase.setdefault("config", {}) _parse_testcase(testcase, project_mapping) - - # unset OS environment variables - utils.unset_os_environ(env_mapping) diff --git a/httprunner/report.py b/httprunner/report.py index 0da55305..af68e429 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -28,6 +28,10 @@ def get_platform(): def get_summary(result): """ get summary from test result + + Args: + result (instance): HtmlTestResult() instance + """ summary = { "success": result.wasSuccessful(), @@ -47,14 +51,11 @@ def get_summary(result): - summary["stat"]["expectedFailures"] \ - summary["stat"]["unexpectedSuccesses"] - if getattr(result, "records", None): - summary["time"] = { - 'start_at': result.start_at, - 'duration': result.duration - } - summary["records"] = result.records - else: - summary["records"] = [] + summary["time"] = { + 'start_at': result.start_at, + 'duration': result.duration + } + summary["records"] = result.records return summary @@ -173,9 +174,8 @@ def stringify_data(meta_data, request_or_response): class HtmlTestResult(unittest.TextTestResult): - """A html result class that can generate formatted html results. - - Used by TextTestRunner. + """ A html result class that can generate formatted html results. + Used by TextTestRunner. """ def __init__(self, stream, descriptions, verbosity): super(HtmlTestResult, self).__init__(stream, descriptions, verbosity) From 31830648859f622a5d7cd0743318c84739c8013c Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 25 Nov 2018 23:12:36 +0800 Subject: [PATCH 039/113] adjust arguments for html report --- httprunner/api.py | 145 +++++++++++++++++++++++++------------------ httprunner/cli.py | 47 ++++++-------- httprunner/loader.py | 2 +- httprunner/report.py | 36 +++++------ tests/test_api.py | 96 ++++++++++++++-------------- 5 files changed, 166 insertions(+), 160 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index 2d17af46..9fc2e999 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -9,21 +9,27 @@ from httprunner import (exceptions, loader, logger, parser, report, runner, class HttpRunner(object): - def __init__(self, **kwargs): + def __init__(self, failfast=False, save_tests=False, report_template=None, report_dir=None): """ initialize HttpRunner. Args: - kwargs (dict): key-value arguments used to initialize TextTestRunner. - Commonly used arguments: - - failfast (bool): False/True, stop the test run on the first error or failure. + failfast (bool): stop the test run on the first error or failure. + save_tests (bool): save loaded/parsed tests to JSON file. + report_template (str): report template file path, template should be in Jinja2 format. + report_dir (str): html report save directory. """ self.exception_stage = "initialize HttpRunner()" - kwargs["resultclass"] = report.HtmlTestResult + kwargs = { + "failfast": failfast, + "resultclass": report.HtmlTestResult + } self.unittest_runner = unittest.TextTestRunner(**kwargs) self.test_loader = unittest.TestLoader() - self.summary = None + self.save_tests = save_tests + self.report_template = report_template + self.report_dir = report_dir + self._summary = None def _add_tests(self, tests_mapping): """ initialize testcase with Runner() and add to test suite. @@ -108,7 +114,7 @@ class HttpRunner(object): tests_results (list): list of (testcase, result) """ - self.summary = { + summary = { "success": True, "stat": {}, "time": {}, @@ -120,83 +126,98 @@ class HttpRunner(object): testcase, result = tests_result testcase_summary = report.get_summary(result) - self.summary["success"] &= testcase_summary["success"] + summary["success"] &= testcase_summary["success"] testcase_summary["name"] = testcase.config.get("name") in_out = utils.get_testcase_io(testcase) utils.print_io(in_out) testcase_summary["in_out"] = in_out - report.aggregate_stat(self.summary["stat"], testcase_summary["stat"]) - report.aggregate_stat(self.summary["time"], testcase_summary["time"]) + report.aggregate_stat(summary["stat"], testcase_summary["stat"]) + report.aggregate_stat(summary["time"], testcase_summary["time"]) - self.summary["details"].append(testcase_summary) + summary["details"].append(testcase_summary) - def run(self, path_or_testcases, dot_env_path=None, mapping=None, save_tests=False): - """ main interface, run testcases with variables mapping. + return summary + + def run_tests(self, tests_mapping): + """ run testcase/testsuite data + """ + # parse tests + self.exception_stage = "parse tests" + parser.parse_tests(tests_mapping) + + if self.save_tests: + utils.dump_tests(tests_mapping, "parsed") + + # add tests to test suite + self.exception_stage = "add tests to test suite" + test_suite = self._add_tests(tests_mapping) + + # run test suite + self.exception_stage = "run test suite" + results = self._run_suite(test_suite) + + # aggregate results + self.exception_stage = "aggregate results" + self._summary = self._aggregate(results) + + # generate html report + self.exception_stage = "generate html report" + report_path = report.render_html_report( + self._summary, + self.report_template, + self.report_dir + ) + + return report_path + + def run_path(self, path, dot_env_path=None, mapping=None): + """ run testcase/testsuite file or folder. Args: - path_or_testcases (str/list/dict): testcase file/foler path, or valid testcases. + path (str): testcase/testsuite file/foler path. dot_env_path (str): specified .env file path. mapping (dict): if mapping is specified, it will override variables in config block. - save_tests (bool): set if save loaded/parsed tests to JSON file. Returns: instance: HttpRunner() instance """ + # load tests self.exception_stage = "load tests" + tests_mapping = loader.load_tests(path, dot_env_path) + tests_mapping["project_mapping"]["test_path"] = path - if validator.is_testcases(path_or_testcases): - tests_mapping = path_or_testcases - elif validator.is_testcase_path(path_or_testcases): - tests_mapping = loader.load_tests(path_or_testcases, dot_env_path) - tests_mapping["project_mapping"]["test_path"] = path_or_testcases - if "variables" in tests_mapping["project_mapping"]: - tests_mapping["project_mapping"]["variables"] = mapping + if mapping: + tests_mapping["project_mapping"]["variables"] = mapping + + if self.save_tests: + utils.dump_tests(tests_mapping, "loaded") + + return self.run_tests(tests_mapping) + + def run(self, path_or_tests, dot_env_path=None, mapping=None): + """ main interface. + + Args: + path_or_tests: + str: testcase/testsuite file/foler path + dict: valid testcase/testsuite data + + """ + if validator.is_testcases(path_or_tests): + return self.run_tests(path_or_tests) + elif validator.is_testcase_path(path_or_tests): + return self.run_path(path_or_tests, dot_env_path, mapping) else: raise exceptions.ParamsError("invalid testcase path or testcases.") - if save_tests: - utils.dump_tests(tests_mapping, "loaded") - - self.exception_stage = "parse tests" - parser.parse_tests(tests_mapping) - - if save_tests: - utils.dump_tests(tests_mapping, "parsed") - - self.exception_stage = "add tests to test suite" - test_suite = self._add_tests(tests_mapping) - - self.exception_stage = "run test suite" - results = self._run_suite(test_suite) - - self.exception_stage = "aggregate results" - self._aggregate(results) - - return self - - def gen_html_report(self, html_report_name=None, html_report_template=None): - """ generate html report and return report path. - - Args: - html_report_name (str): output html report file name - html_report_template (str): report template file path, template should be in Jinja2 format - - Returns: - str: generated html report path - + @property + def summary(self): + """ get test reuslt summary. """ - if not self.summary: - raise exceptions.MyBaseError("run method should be called before gen_html_report.") - - self.exception_stage = "generate report" - return report.render_html_report( - self.summary, - html_report_name, - html_report_template - ) + return self._summary def prepare_locust_tests(path): diff --git a/httprunner/cli.py b/httprunner/cli.py index 7c928535..bb8ec9d8 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -24,15 +24,6 @@ def main_hrun(): parser.add_argument( 'testcase_paths', nargs='*', help="testcase file path") - parser.add_argument( - '--no-html-report', action='store_true', default=False, - help="do not generate html report.") - parser.add_argument( - '--html-report-name', - help="specify html report name, only effective when generating html report.") - parser.add_argument( - '--html-report-template', - help="specify html report template path.") parser.add_argument( '--log-level', default='INFO', help="Specify logging level, default is INFO.") @@ -42,6 +33,12 @@ def main_hrun(): parser.add_argument( '--dot-env-path', help="Specify .env file path, which is useful for keeping sensitive data.") + parser.add_argument( + '--report-template', + help="specify report template path.") + parser.add_argument( + '--report-dir', + help="specify report save directory.") parser.add_argument( '--failfast', action='store_true', default=False, help="Stop the test run on the first error or failure.") @@ -80,26 +77,20 @@ def main_hrun(): create_scaffold(project_name) exit(0) - for path in args.testcase_paths: - try: - runner = HttpRunner(failfast=args.failfast) - runner.run( - path, - dot_env_path=args.dot_env_path, - save_tests=args.save_tests - ) - except Exception: - logger.log_error("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage)) - raise + runner = HttpRunner( + failfast=args.failfast, + save_tests=args.save_tests, + report_template=args.report_template, + report_dir=args.report_dir + ) + try: + for path in args.testcase_paths: + runner.run(path, dot_env_path=args.dot_env_path) + except Exception: + logger.log_error("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage)) + raise - if not args.no_html_report: - runner.gen_html_report( - html_report_name=args.html_report_name, - html_report_template=args.html_report_template - ) - - summary = runner.summary - return 0 if summary["success"] else 1 + return 0 def main_locust(): """ Performance test with locust: parse command line options and run commands. diff --git a/httprunner/loader.py b/httprunner/loader.py index d03cbde8..6f147a53 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -585,7 +585,7 @@ def load_tests(path, dot_env_path=None): """ load testcases from file path, extend and merge with api/testcase definitions. Args: - path (str/list): testcase file/foler path. + path (str): testcase/testsuite file/foler path. path could be in 2 types: - absolute/relative file path - absolute/relative folder path diff --git a/httprunner/report.py b/httprunner/report.py index af68e429..5450541b 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -78,37 +78,36 @@ def aggregate_stat(origin_stat, new_stat): origin_stat[key] += new_stat[key] -def render_html_report(summary, html_report_name=None, html_report_template=None): +def render_html_report(summary, report_template=None, report_dir=None): """ render html report with specified report name and template - if html_report_name is not specified, use current datetime - if html_report_template is not specified, use default report template + + Args: + report_template (str): specify html report template path + report_dir (str): specify html report save directory + """ - if not html_report_template: - html_report_template = os.path.join( + if not report_template: + report_template = os.path.join( os.path.abspath(os.path.dirname(__file__)), "templates", "report_template.html" ) logger.log_debug("No html report template specified, use default.") else: - logger.log_info("render with html report template: {}".format(html_report_template)) + logger.log_info("render with html report template: {}".format(report_template)) logger.log_info("Start to render Html report ...") logger.log_debug("render data: {}".format(summary)) - report_dir_path = os.path.join(os.getcwd(), "reports") + report_dir = report_dir or os.path.join(os.getcwd(), "reports") + if not os.path.isdir(report_dir): + os.makedirs(report_dir) + start_at_timestamp = int(summary["time"]["start_at"]) summary["time"]["start_datetime"] = datetime.fromtimestamp(start_at_timestamp).strftime('%Y-%m-%d %H:%M:%S') - if html_report_name: - summary["html_report_name"] = html_report_name - report_dir_path = os.path.join(report_dir_path, html_report_name) - html_report_name += "-{}.html".format(start_at_timestamp) - else: - summary["html_report_name"] = "" - html_report_name = "{}.html".format(start_at_timestamp) - - if not os.path.isdir(report_dir_path): - os.makedirs(report_dir_path) + summary["html_report_name"] = "" + html_report_name = "{}.html".format(start_at_timestamp) + report_path = os.path.join(report_dir, html_report_name) for index, suite_summary in enumerate(summary["details"]): if not suite_summary.get("name"): @@ -118,9 +117,8 @@ def render_html_report(summary, html_report_name=None, html_report_template=None stringify_data(meta_data, 'request') stringify_data(meta_data, 'response') - with io.open(html_report_template, "r", encoding='utf-8') as fp_r: + with io.open(report_template, "r", encoding='utf-8') as fp_r: template_content = fp_r.read() - report_path = os.path.join(report_dir_path, html_report_name) with io.open(report_path, 'w', encoding='utf-8') as fp_w: rendered_content = Template( template_content, diff --git a/tests/test_api.py b/tests/test_api.py index a1cb4649..d46ec0a3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -65,6 +65,7 @@ class TestHttpRunner(ApiServerUnittest): self.tests_mapping = { "testcases": testcases } + self.runner = HttpRunner(failfast=True) self.reset_all() def reset_all(self): @@ -73,36 +74,34 @@ class TestHttpRunner(ApiServerUnittest): return self.api_client.get(url, headers=headers) def test_text_run_times(self): - runner = HttpRunner().run(self.testcase_cli_path) - self.assertEqual(runner.summary["stat"]["testsRun"], 10) + self.runner.run(self.testcase_cli_path) + self.assertEqual(self.runner.summary["stat"]["testsRun"], 10) def test_text_skip(self): - runner = HttpRunner().run(self.testcase_cli_path) - self.assertEqual(runner.summary["stat"]["skipped"], 4) + self.runner.run(self.testcase_cli_path) + self.assertEqual(self.runner.summary["stat"]["skipped"], 4) def test_html_report(self): - runner = HttpRunner().run(self.testcase_cli_path) + report_save_dir = os.path.join(os.getcwd(), 'reports', "demo") + runner = HttpRunner(failfast=True, report_dir=report_save_dir) + runner.run(self.testcase_cli_path) summary = runner.summary self.assertEqual(summary["stat"]["testsRun"], 10) self.assertEqual(summary["stat"]["skipped"], 4) - - output_folder_name = "demo" - runner.gen_html_report(html_report_name=output_folder_name) - report_save_dir = os.path.join(os.getcwd(), 'reports', output_folder_name) self.assertGreater(len(os.listdir(report_save_dir)), 0) shutil.rmtree(report_save_dir) def test_run_testcases(self): - runner = HttpRunner().run(self.tests_mapping) - summary = runner.summary + self.runner.run_tests(self.tests_mapping) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 2) self.assertIn("details", summary) self.assertIn("records", summary["details"][0]) def test_run_yaml_upload(self): - runner = HttpRunner().run("tests/httpbin/upload.yml") - summary = runner.summary + self.runner.run("tests/httpbin/upload.yml") + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 1) self.assertIn("details", summary) @@ -140,31 +139,29 @@ class TestHttpRunner(ApiServerUnittest): tests_mapping = { "testcases": testcases } - runner = HttpRunner().run(tests_mapping) - summary = runner.summary + self.runner.run_tests(tests_mapping) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 1) self.assertEqual(summary["details"][0]["records"][0]["meta_data"]["response"]["json"]["data"], "abc") def test_html_report_repsonse_image(self): - runner = HttpRunner().run("tests/httpbin/load_image.yml") - summary = runner.summary - output_folder_name = "demo" - report = runner.gen_html_report(html_report_name=output_folder_name) + report_save_dir = os.path.join(os.getcwd(), 'reports', "demo") + runner = HttpRunner(failfast=True, report_dir=report_save_dir) + report = runner.run("tests/httpbin/load_image.yml") self.assertTrue(os.path.isfile(report)) - report_save_dir = os.path.join(os.getcwd(), 'reports', output_folder_name) shutil.rmtree(report_save_dir) def test_testcase_layer_with_api(self): - runner = HttpRunner(failfast=True).run("tests/testcases/setup.yml") - summary = runner.summary + self.runner.run("tests/testcases/setup.yml") + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["details"][0]["records"][0]["name"], "get token (setup)") self.assertEqual(summary["stat"]["testsRun"], 2) def test_testcase_layer_with_testcase(self): - runner = HttpRunner(failfast=True).run("tests/testsuites/create_users.yml") - summary = runner.summary + self.runner.run("tests/testsuites/create_users.yml") + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 2) @@ -172,9 +169,9 @@ class TestHttpRunner(ApiServerUnittest): testcase_file_path = os.path.join( os.getcwd(), 'tests/httpbin/hooks.yml') start_time = time.time() - runner = HttpRunner().run(testcase_file_path) + self.runner.run(testcase_file_path) end_time = time.time() - summary = runner.summary + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertLess(end_time - start_time, 60) @@ -212,8 +209,8 @@ class TestHttpRunner(ApiServerUnittest): "project_mapping": loader.project_mapping, "testcases": testcases } - runner = HttpRunner().run(tests_mapping) - summary = runner.summary + self.runner.run_tests(tests_mapping) + summary = self.runner.summary self.assertTrue(summary["success"]) def test_run_httprunner_with_teardown_hooks_not_exist_attribute(self): @@ -245,8 +242,8 @@ class TestHttpRunner(ApiServerUnittest): "project_mapping": loader.project_mapping, "testcases": testcases } - runner = HttpRunner().run(tests_mapping) - summary = runner.summary + self.runner.run_tests(tests_mapping) + summary = self.runner.summary self.assertFalse(summary["success"]) self.assertEqual(summary["stat"]["errors"], 1) @@ -276,15 +273,15 @@ class TestHttpRunner(ApiServerUnittest): "project_mapping": loader.project_mapping, "testcases": testcases } - runner = HttpRunner().run(tests_mapping) - summary = runner.summary + self.runner.run_tests(tests_mapping) + summary = self.runner.summary self.assertFalse(summary["success"]) self.assertEqual(summary["stat"]["errors"], 1) def test_run_testcase_hardcode(self): for testcase_file_path in self.testcase_file_path_list: - runner = HttpRunner().run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testsRun"], 3) self.assertEqual(summary["stat"]["successes"], 3) @@ -292,30 +289,30 @@ class TestHttpRunner(ApiServerUnittest): def test_run_testcase_template_variables(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_variables.yml') - runner = HttpRunner().run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary self.assertTrue(summary["success"]) def test_run_testcase_template_import_functions(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_functions.yml') - runner = HttpRunner().run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary self.assertTrue(summary["success"]) def test_run_testcase_layered(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_layer.yml') - runner = HttpRunner().run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(len(summary["details"]), 1) def test_run_testcase_output(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_testcase_layer.yml') - runner = HttpRunner(failfast=True).run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertIn("token", summary["details"][0]["in_out"]["out"]) # TODO: add @@ -327,8 +324,8 @@ class TestHttpRunner(ApiServerUnittest): variables_mapping = { "app_version": '2.9.7' } - runner = HttpRunner(failfast=True).run(testcase_file_path, mapping=variables_mapping) - summary = runner.summary + self.runner.run(testcase_file_path, mapping=variables_mapping) + summary = self.runner.summary self.assertTrue(summary["success"]) self.assertIn("token", summary["details"][0]["in_out"]["out"]) # TODO: add @@ -337,8 +334,8 @@ class TestHttpRunner(ApiServerUnittest): def test_run_testcase_with_parameters(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/demo_parameters.yml') - runner = HttpRunner().run(testcase_file_path) - summary = runner.summary + self.runner.run(testcase_file_path) + summary = self.runner.summary # TODO: add parameterize # self.assertEqual( # summary["details"][0]["in_out"]["in"]["user_agent"], @@ -364,8 +361,7 @@ class TestHttpRunner(ApiServerUnittest): os.getcwd(), 'tests/data/demo_parameters.yml') tests_mapping = loader.load_tests(testcase_file_path) parser.parse_tests(tests_mapping) - runner = HttpRunner() - test_suite = runner._add_tests(tests_mapping) + test_suite = self.runner._add_tests(tests_mapping) self.assertEqual( test_suite._tests[0].tests[0]['name'], @@ -396,8 +392,8 @@ class TestHttpRunner(ApiServerUnittest): def test_validate_response_content(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/httpbin/basic.yml') - runner = HttpRunner().run(testcase_file_path) - self.assertTrue(runner.summary["success"]) + self.runner.run(testcase_file_path) + self.assertTrue(self.runner.summary["success"]) class TestApi(ApiServerUnittest): From 680ed1a1baedb82967c3a33866491ac3d5504b61 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 25 Nov 2018 23:52:11 +0800 Subject: [PATCH 040/113] dump test result summary to json file --- httprunner/api.py | 3 +++ httprunner/utils.py | 52 ++++++++++++++++++++++++++++++++------------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index 9fc2e999..35810d55 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -162,6 +162,9 @@ class HttpRunner(object): self.exception_stage = "aggregate results" self._summary = self._aggregate(results) + if self.save_tests: + utils.dump_summary(self._summary, tests_mapping["project_mapping"]) + # generate html report self.exception_stage = "generate html report" report_path = report.render_html_report( diff --git a/httprunner/utils.py b/httprunner/utils.py index 20f6f2eb..fcfeac01 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -604,9 +604,36 @@ def prettify_json_file(file_list): print("success: {}".format(outfile)) +def dump_json_file(json_data, pwd_dir_path, dump_file_name): + """ dump json data to file + """ + logs_dir_path = os.path.join(pwd_dir_path, "logs") + if not os.path.isdir(logs_dir_path): + os.makedirs(logs_dir_path) + + dump_file_path = os.path.join(logs_dir_path, dump_file_name) + + with open(dump_file_path, 'w', encoding='utf-8') as outfile: + json.dump(json_data, outfile, indent=4, separators=(',', ': ')) + + msg = "dump file: {}".format(dump_file_path) + logger.color_print(msg, "BLUE") + + +def _prepare_dump_info(project_mapping, tag_name): + """ prepare dump file info. + """ + test_path = project_mapping.get("test_path") or "tests_mapping" + pwd_dir_path = project_mapping.get("PWD") or os.getcwd() + file_name, file_suffix = os.path.splitext(os.path.basename(test_path)) + dump_file_name = "{}.{}.json".format(file_name, tag_name) + + return pwd_dir_path, dump_file_name + + def dump_tests(tests_mapping, tag_name): - """ dump all tests data (except functions) to json file. - the dumped files are located in PWD/logs folder. + """ dump loaded/parsed tests data (except functions) to json file. + the dumped file is located in PWD/logs folder. Args: tests_mapping (dict): data to dump @@ -614,15 +641,7 @@ def dump_tests(tests_mapping, tag_name): """ project_mapping = tests_mapping.get("project_mapping", {}) - test_path = project_mapping.get("test_path") or "tests_mapping" - pwd_dir_path = project_mapping.get("PWD") or os.getcwd() - file_name, file_suffix = os.path.splitext(os.path.basename(test_path)) - - logs_dir_path = os.path.join(pwd_dir_path, "logs") - if not os.path.isdir(logs_dir_path): - os.makedirs(logs_dir_path) - - dump_file_path = os.path.join(logs_dir_path, "{}.{}.json".format(file_name, tag_name)) + pwd_dir_path, dump_file_name = _prepare_dump_info(project_mapping, tag_name) tests_to_dump = { "project_mapping": {} @@ -639,11 +658,14 @@ def dump_tests(tests_mapping, tag_name): tests_to_dump["testcases"] = tests_mapping["testcases"] - with open(dump_file_path, 'w', encoding='utf-8') as outfile: - json.dump(tests_to_dump, outfile, indent=4, separators=(',', ': ')) + dump_json_file(tests_to_dump, pwd_dir_path, dump_file_name) - msg = "{} file generated successfully: {}".format(tag_name, dump_file_path) - logger.color_print(msg, "BLUE") + +def dump_summary(summary, project_mapping): + """ dump test result summary to json file. + """ + pwd_dir_path, dump_file_name = _prepare_dump_info(project_mapping, "summary") + dump_json_file(summary, pwd_dir_path, dump_file_name) def get_python2_retire_msg(): From 1b9b4b559230eb65cb7ed020c12d7e500cd8338d Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 26 Nov 2018 11:23:48 +0800 Subject: [PATCH 041/113] fix generated log filename when testing folder with '/' ending --- httprunner/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/httprunner/utils.py b/httprunner/utils.py index fcfeac01..81f2ba41 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -415,13 +415,13 @@ def get_testcase_io(testcase): dict: input(variables) and output mapping. """ - runner = testcase.runner - variables = testcase.config.get("variables", []) + test_runner = testcase.runner + variables = testcase.config.get("variables", {}) output_list = testcase.config.get("output", []) return { - "in": dict(variables), - "out": runner.extract_output(output_list) + "in": variables, + "out": test_runner.extract_output(output_list) } @@ -625,7 +625,7 @@ def _prepare_dump_info(project_mapping, tag_name): """ test_path = project_mapping.get("test_path") or "tests_mapping" pwd_dir_path = project_mapping.get("PWD") or os.getcwd() - file_name, file_suffix = os.path.splitext(os.path.basename(test_path)) + file_name, file_suffix = os.path.splitext(os.path.basename(test_path.rstrip("/"))) dump_file_name = "{}.{}.json".format(file_name, tag_name) return pwd_dir_path, dump_file_name From ead4332ee2b0f2ad3cfe2338d73e79a4a2e88944 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 26 Nov 2018 11:50:55 +0800 Subject: [PATCH 042/113] remove render summary data --- httprunner/report.py | 1 - 1 file changed, 1 deletion(-) diff --git a/httprunner/report.py b/httprunner/report.py index 5450541b..b8dfa8ef 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -97,7 +97,6 @@ def render_html_report(summary, report_template=None, report_dir=None): logger.log_info("render with html report template: {}".format(report_template)) logger.log_info("Start to render Html report ...") - logger.log_debug("render data: {}".format(summary)) report_dir = report_dir or os.path.join(os.getcwd(), "reports") if not os.path.isdir(report_dir): From c7a764bfc7618cba8766496ab871fff693aeba65 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 26 Nov 2018 13:44:33 +0800 Subject: [PATCH 043/113] fix: stringify summary before dump json file --- httprunner/api.py | 6 ++++-- httprunner/report.py | 32 +++++++++++++++++++------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index 35810d55..daef4269 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -162,11 +162,13 @@ class HttpRunner(object): self.exception_stage = "aggregate results" self._summary = self._aggregate(results) + # generate html report + self.exception_stage = "generate html report" + report.stringify_summary(self._summary) + if self.save_tests: utils.dump_summary(self._summary, tests_mapping["project_mapping"]) - # generate html report - self.exception_stage = "generate html report" report_path = report.render_html_report( self._summary, self.report_template, diff --git a/httprunner/report.py b/httprunner/report.py index b8dfa8ef..15b4d12c 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -78,6 +78,24 @@ def aggregate_stat(origin_stat, new_stat): origin_stat[key] += new_stat[key] +def stringify_summary(summary): + """ stringify summary, in order to dump json file and generate html report. + """ + start_at_timestamp = int(summary["time"]["start_at"]) + summary["time"]["start_datetime"] = datetime.fromtimestamp(start_at_timestamp).strftime('%Y-%m-%d %H:%M:%S') + summary["html_report_name"] = "{}.html".format(start_at_timestamp) + + for index, suite_summary in enumerate(summary["details"]): + + if not suite_summary.get("name"): + suite_summary["name"] = "test suite {}".format(index) + + for record in suite_summary.get("records"): + meta_data = record['meta_data'] + stringify_data(meta_data, 'request') + stringify_data(meta_data, 'response') + + def render_html_report(summary, report_template=None, report_dir=None): """ render html report with specified report name and template @@ -102,19 +120,7 @@ def render_html_report(summary, report_template=None, report_dir=None): if not os.path.isdir(report_dir): os.makedirs(report_dir) - start_at_timestamp = int(summary["time"]["start_at"]) - summary["time"]["start_datetime"] = datetime.fromtimestamp(start_at_timestamp).strftime('%Y-%m-%d %H:%M:%S') - summary["html_report_name"] = "" - html_report_name = "{}.html".format(start_at_timestamp) - report_path = os.path.join(report_dir, html_report_name) - - for index, suite_summary in enumerate(summary["details"]): - if not suite_summary.get("name"): - suite_summary["name"] = "test suite {}".format(index) - for record in suite_summary.get("records"): - meta_data = record['meta_data'] - stringify_data(meta_data, 'request') - stringify_data(meta_data, 'response') + report_path = os.path.join(report_dir, summary["html_report_name"]) with io.open(report_template, "r", encoding='utf-8') as fp_r: template_content = fp_r.read() From 0a44aa8bfe85aa39a1b33d9c16177dda1918b637 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 26 Nov 2018 14:26:30 +0800 Subject: [PATCH 044/113] fix html report display response content --- httprunner/client.py | 30 ++++++++++++++--------- httprunner/templates/report_template.html | 8 +++--- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/httprunner/client.py b/httprunner/client.py index c62d12aa..e1798db9 100644 --- a/httprunner/client.py +++ b/httprunner/client.py @@ -138,20 +138,26 @@ class HttpSession(requests.Session): self.meta_data["response"]["headers"] = dict(response.headers) self.meta_data["response"]["cookies"] = response.cookies or {} self.meta_data["response"]["encoding"] = response.encoding - self.meta_data["response"]["content_type"] = response.headers.get("Content-Type", "") - try: - # try to only record json data - self.meta_data["response"]["json"] = response.json() - except ValueError: - # only record at most 1000 text charactors - resp_text = response.text - resp_text_length = len(resp_text) - if resp_text_length > 1000: - resp_text = resp_text[0:1000] \ - + " ... OMITTED {} CHARACTORS ...".format(resp_text_length-1000) + content_type = response.headers.get("Content-Type", "") + self.meta_data["response"]["content_type"] = content_type - self.meta_data["response"]["text"] = resp_text + if "image" in content_type: + # response is image type, record bytes content only + self.meta_data["response"]["content"] = response.content + else: + try: + # try to record json data + self.meta_data["response"]["json"] = response.json() + except ValueError: + # only record at most 1000 text charactors + resp_text = response.text + resp_text_length = len(resp_text) + if resp_text_length > 1000: + resp_text = resp_text[0:1000] \ + + " ... OMITTED {} CHARACTORS ...".format(resp_text_length-1000) + + self.meta_data["response"]["text"] = resp_text # get the length of the content, but if the argument stream is set to True, we take # the size from the content-length header, in order to not trigger fetching of the body diff --git a/httprunner/templates/report_template.html b/httprunner/templates/report_template.html index 7cca19cb..8b68a425 100644 --- a/httprunner/templates/report_template.html +++ b/httprunner/templates/report_template.html @@ -272,7 +272,7 @@
{% for key, value in record.meta_data.response.items() %} - {% if key in ["text", "json", "elapsed_ms", "response_time_ms", "content_size", "content_type"] %} + {% if key in ["elapsed_ms", "response_time_ms", "content_size", "content_type"] %} {% continue %} {% endif %} @@ -288,10 +288,12 @@ {% if "image" in record.meta_data.response.content_type %} {% else %} - 
{{ record.meta_data.response.text | e }}
+ {{ value }} {% endif %} + {% elif key == "text" %} + 
{{ record.meta_data.response.text | e }}
{% else %} - {{value}} + {{ value }} {% endif %} From 5539b9e57477c6831fc29b287ebecf009d483337 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 26 Nov 2018 17:39:25 +0800 Subject: [PATCH 045/113] fix compatibility with locust --- httprunner/runner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/httprunner/runner.py b/httprunner/runner.py index 7dd05360..6ea969c3 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -71,12 +71,18 @@ class Runner(object): def clear_test_data(self): """ clear request and response data """ + if not isinstance(self.http_client_session, HttpSession): + return + self.evaluated_validators = [] self.http_client_session.init_meta_data() def get_test_data(self): """ get request/response data and validate results """ + if not isinstance(self.http_client_session, HttpSession): + raise exceptions.FunctionNotFound("get_test_data is only valid in HttpSession!") + meta_data = self.http_client_session.meta_data meta_data["validators"] = self.evaluated_validators return meta_data From 6e122ec8ac0ecc077447ef16f9de8e31a77885e8 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 27 Nov 2018 17:55:49 +0800 Subject: [PATCH 046/113] fix html report for HttpRunner 2.0 --- httprunner/api.py | 2 +- httprunner/client.py | 1 - httprunner/report.py | 208 +++++++++++++++++----- httprunner/runner.py | 18 +- httprunner/templates/report_template.html | 34 ++-- tests/test_api.py | 2 +- 6 files changed, 197 insertions(+), 68 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index daef4269..a9048750 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -50,7 +50,7 @@ class HttpRunner(object): except exceptions.MyBaseFailure as ex: self.fail(str(ex)) finally: - self.meta_data = test_runner.get_test_data() + self.meta_datas = test_runner.meta_datas if "config" in test_dict: # run nested testcase diff --git a/httprunner/client.py b/httprunner/client.py index e1798db9..d540fcb1 100644 --- a/httprunner/client.py +++ b/httprunner/client.py @@ -125,7 +125,6 @@ class HttpSession(requests.Session): # record actual request info self.meta_data["request"]["url"] = (response.history and response.history[0] or response).request.url self.meta_data["request"]["headers"] = dict(response.request.headers) - self.meta_data["request"]["body"] = response.request.body # log request details in debug mode log_print("request") diff --git a/httprunner/report.py b/httprunner/report.py index 15b4d12c..65dbd183 100644 --- a/httprunner/report.py +++ b/httprunner/report.py @@ -91,9 +91,169 @@ def stringify_summary(summary): suite_summary["name"] = "test suite {}".format(index) for record in suite_summary.get("records"): - meta_data = record['meta_data'] - stringify_data(meta_data, 'request') - stringify_data(meta_data, 'response') + meta_datas = record['meta_datas'] + __stringify_meta_datas(meta_datas) + meta_datas_expanded = [] + __expand_meta_datas(meta_datas, meta_datas_expanded) + record["meta_datas_expanded"] = meta_datas_expanded + record["response_time"] = __get_total_response_time(meta_datas_expanded) + + +def __stringify_request(request_data): + """ stringfy HTTP request data + + Args: + request_data (dict): HTTP request data in dict. + + { + "url": "http://127.0.0.1:5000/api/get-token", + "method": "POST", + "headers": { + "User-Agent": "python-requests/2.20.0", + "Accept-Encoding": "gzip, deflate", + "Accept": "*/*", + "Connection": "keep-alive", + "user_agent": "iOS/10.3", + "device_sn": "TESTCASE_CREATE_XXX", + "os_platform": "ios", + "app_version": "2.8.6", + "Content-Type": "application/json", + "Content-Length": "52" + }, + "start_timestamp": 1543299567.6505039, + "json": { + "sign": "cb9d60acd09080ea66c8e63a1c78c6459ea00168" + }, + "verify": false + } + + """ + for key, value in request_data.items(): + + if isinstance(value, list): + value = json.dumps(value, indent=2, ensure_ascii=False) + + elif isinstance(value, bytes): + try: + encoding = "utf-8" + value = escape(value.decode(encoding)) + except UnicodeDecodeError: + pass + + elif not isinstance(value, (basestring, numeric_types, Iterable)): + # class instance, e.g. MultipartEncoder() + value = repr(value) + + request_data[key] = value + + +def __stringify_response(response_data): + """ stringfy HTTP response data + + Args: + response_data (dict): + + { + "status_code": 404, + "headers": { + "Content-Type": "application/json", + "Content-Length": "30", + "Server": "Werkzeug/0.14.1 Python/3.7.0", + "Date": "Tue, 27 Nov 2018 06:19:27 GMT" + }, + "content_size": 30, + "response_time_ms": 3.63, + "elapsed_ms": 2.197, + "encoding": "None", + "content_type": "application/json", + "ok": false, + "url": "http://127.0.0.1:5000/api/users/9001", + "reason": "NOT FOUND", + "cookies": {}, + "json": { + "success": false, + "data": {} + } + } + + """ + for key, value in response_data.items(): + + if isinstance(value, list): + value = json.dumps(value, indent=2, ensure_ascii=False) + + elif isinstance(value, bytes): + try: + encoding = response_data.get("encoding") + if not encoding or encoding == "None": + encoding = "utf-8" + + if key == "content" and "image" in response_data["content_type"]: + # display image + value = "data:{};base64,{}".format( + response_data["content_type"], + b64encode(value).decode(encoding) + ) + else: + value = escape(value.decode(encoding)) + except UnicodeDecodeError: + pass + + elif not isinstance(value, (basestring, numeric_types, Iterable)): + # class instance, e.g. MultipartEncoder() + value = repr(value) + + response_data[key] = value + + +def __expand_meta_datas(meta_datas, meta_datas_expanded): + """ expand meta_datas to one level + + Args: + meta_datas (dict/list): maybe in nested format + + Returns: + list: expanded list in one level + + Examples: + >>> meta_datas = [ + [ + dict1, + dict2 + ], + dict3 + ] + >>> meta_datas_expanded = [] + >>> __expand_meta_datas(meta_datas, meta_datas_expanded) + >>> print(meta_datas_expanded) + [dict1, dict2, dict3] + + """ + if isinstance(meta_datas, dict): + meta_datas_expanded.append(meta_datas) + elif isinstance(meta_datas, list): + for meta_data in meta_datas: + __expand_meta_datas(meta_data, meta_datas_expanded) + + +def __get_total_response_time(meta_datas_expanded): + """ caculate total response time of all meta_datas + """ + response_time = 0 + for meta_data in meta_datas_expanded: + response_time += meta_data["response"]["response_time_ms"] + + return "{:.2f}".format(response_time) + + +def __stringify_meta_datas(meta_datas): + + if isinstance(meta_datas, list): + for _meta_data in meta_datas: + __stringify_meta_datas(_meta_data) + elif isinstance(meta_datas, dict): + __stringify_request(meta_datas["request"]) + __stringify_response(meta_datas["response"]) def render_html_report(summary, report_template=None, report_dir=None): @@ -136,46 +296,6 @@ def render_html_report(summary, report_template=None, report_dir=None): return report_path -def stringify_data(meta_data, request_or_response): - """ - meta_data = { - "request": {}, - "response": {} - } - """ - headers = meta_data[request_or_response]["headers"] - request_or_response_dict = meta_data[request_or_response] - - for key, value in request_or_response_dict.items(): - - if isinstance(value, list): - value = json.dumps(value, indent=2, ensure_ascii=False) - - elif isinstance(value, bytes): - try: - encoding = meta_data["response"].get("encoding") - if not encoding or encoding == "None": - encoding = "utf-8" - - if request_or_response == "response" and key == "content" \ - and "image" in meta_data["response"]["content_type"]: - # display image - value = "data:{};base64,{}".format( - meta_data["response"]["content_type"], - b64encode(value).decode(encoding) - ) - else: - value = escape(value.decode(encoding)) - except UnicodeDecodeError: - pass - - elif not isinstance(value, (basestring, numeric_types, Iterable)): - # class instance, e.g. MultipartEncoder() - value = repr(value) - - meta_data[request_or_response][key] = value - - class HtmlTestResult(unittest.TextTestResult): """ A html result class that can generate formatted html results. Used by TextTestRunner. @@ -189,7 +309,7 @@ class HtmlTestResult(unittest.TextTestResult): 'name': test.shortDescription(), 'status': status, 'attachment': attachment, - "meta_data": test.meta_data + "meta_datas": test.meta_datas } self.records.append(data) diff --git a/httprunner/runner.py b/httprunner/runner.py index 6ea969c3..9fef5302 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -68,7 +68,7 @@ class Runner(object): if self.testcase_teardown_hooks: self.do_hook_actions(self.testcase_teardown_hooks, "teardown") - def clear_test_data(self): + def __clear_test_data(self): """ clear request and response data """ if not isinstance(self.http_client_session, HttpSession): @@ -77,11 +77,11 @@ class Runner(object): self.evaluated_validators = [] self.http_client_session.init_meta_data() - def get_test_data(self): + def __get_test_data(self): """ get request/response data and validate results """ if not isinstance(self.http_client_session, HttpSession): - raise exceptions.FunctionNotFound("get_test_data is only valid in HttpSession!") + return meta_data = self.http_client_session.meta_data meta_data["validators"] = self.evaluated_validators @@ -167,7 +167,7 @@ class Runner(object): "authorization": "$authorization", "random": "$random" }, - "body": '{"name": "user", "password": "123456"}' + "json": {"name": "user", "password": "123456"} }, "extract": [], # optional "validate": [], # optional @@ -182,7 +182,7 @@ class Runner(object): """ # clear meta data first to ensure independence for each test - self.clear_test_data() + self.__clear_test_data() # check skip self._handle_skip_feature(test_dict) @@ -266,14 +266,18 @@ class Runner(object): def _run_testcase(self, testcase_dict): """ run single testcase. """ + meta_data_list = [] config = testcase_dict.get("config", {}) test_runner = Runner(config, self.functions, self.http_client_session) tests = testcase_dict.get("tests", []) for index, test_dict in enumerate(tests): test_runner.run_test(test_dict) + meta_datas = test_runner.meta_datas + meta_data_list.append(meta_datas) self.session_context.update_seesion_variables(test_runner.extract_sessions()) + return meta_data_list def run_test(self, test_dict): """ run single teststep of testcase. @@ -308,12 +312,14 @@ class Runner(object): } """ + self.meta_datas = None if "config" in test_dict: # nested testcase - self._run_testcase(test_dict) + self.meta_datas = self._run_testcase(test_dict) else: # api self._run_test(test_dict) + self.meta_datas = self.__get_test_data() def extract_sessions(self): """ diff --git a/httprunner/templates/report_template.html b/httprunner/templates/report_template.html index 8b68a425..9b53e5fd 100644 --- a/httprunner/templates/report_template.html +++ b/httprunner/templates/report_template.html @@ -233,28 +233,31 @@ {% for record in test_suite_summary.records %} {% set record_index = "{}_{}".format(suite_index, loop.index) %} + {% set record_meta_datas = record.meta_datas_expanded %} - +
{{record.status}} {{record.name}}{{ record.meta_data.response.response_time_ms }} ms{{ record.response_time }} ms - log -