From 597d40537850a91804c3cd30d9199a0568881b50 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 29 Oct 2018 13:01:30 +0800 Subject: [PATCH 1/5] refactor _load_test_file --- httprunner/loader.py | 29 +++++++++++++++-------------- tests/test_loader.py | 3 ++- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index d7f51502..59b8203a 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -315,11 +315,11 @@ def get_module_item(module_mapping, item_type, item_name): ## testcase loader ############################################################################### -def _load_test_file(file_path, project_mapping): - """ load testcase file or testsuite file +def _load_testcase(raw_testcase, project_mapping): + """ load testcase/testsuite with api/testcase references Args: - file_path (str): absolute valid file path. file_path should be in the following format: + raw_testcase (list): raw testcase content loaded from JSON/YAML file: [ { "config": { @@ -353,29 +353,29 @@ def _load_test_file(file_path, project_mapping): project_mapping (dict): project_mapping Returns: - dict: testcase dict + dict: loaded testcase content { "config": {}, "teststeps": [teststep11, teststep12] } """ - testcase = { + loaded_testcase = { "config": {}, "teststeps": [] } - for item in load_file(file_path): + for item in raw_testcase: # TODO: add json schema validation if not isinstance(item, dict) or len(item) != 1: - raise exceptions.FileFormatError("Testcase format error: {}".format(file_path)) + raise exceptions.FileFormatError("Testcase format error: {}".format(item)) key, test_block = item.popitem() if not isinstance(test_block, dict): - raise exceptions.FileFormatError("Testcase format error: {}".format(file_path)) + raise exceptions.FileFormatError("Testcase format error: {}".format(item)) if key == "config": - testcase["config"].update(test_block) + loaded_testcase["config"].update(test_block) elif key == "test": @@ -387,7 +387,7 @@ def _load_test_file(file_path, project_mapping): # reference api if "api" in test_block: extend_api_definition(test_block) - testcase["teststeps"].append(test_block) + loaded_testcase["teststeps"].append(test_block) # reference testcase elif "suite" in test_block: # TODO: replace suite with testcase @@ -397,18 +397,18 @@ def _load_test_file(file_path, project_mapping): for teststep in block["teststeps"]: if "api" in teststep: extend_api_definition(teststep) - testcase["teststeps"].append(teststep) + loaded_testcase["teststeps"].append(teststep) # define directly else: - testcase["teststeps"].append(test_block) + loaded_testcase["teststeps"].append(test_block) else: logger.log_warning( "unexpected block key: {}. block key should only be 'config' or 'test'.".format(key) ) - return testcase + return loaded_testcase def _get_block_by_name(ref_call, ref_type, project_mapping): @@ -956,8 +956,9 @@ def load_tests(path, dot_env_path=None): elif os.path.isfile(path): try: + raw_testcase = load_file(path) project_mapping = load_project_tests(path, dot_env_path) - testcase = _load_test_file(path, project_mapping) + testcase = _load_testcase(raw_testcase, project_mapping) testcase["config"]["path"] = path testcase["config"]["refs"] = project_mapping testcases_list = [testcase] diff --git a/tests/test_loader.py b/tests/test_loader.py index ff2bd996..ebeff85c 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -284,7 +284,8 @@ class TestSuiteLoader(unittest.TestCase): cls.project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) def test_load_test_file_testcase(self): - testcase = loader._load_test_file("tests/testcases/smoketest.yml", self.project_mapping) + raw_testcase = loader.load_file("tests/testcases/smoketest.yml") + testcase = loader._load_testcase(raw_testcase, self.project_mapping) self.assertEqual(testcase["config"]["name"], "smoketest") self.assertIn("device_sn", testcase["config"]["variables"][0]) self.assertEqual(len(testcase["teststeps"]), 8) From aed7013b1d04aebeae2d52a7098ee01afef0dac9 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 29 Oct 2018 14:53:20 +0800 Subject: [PATCH 2/5] split _load_teststeps from _load_testcase --- httprunner/loader.py | 105 ++++++++++++++++++++++++++----------------- tests/test_loader.py | 13 +++++- 2 files changed, 75 insertions(+), 43 deletions(-) diff --git a/httprunner/loader.py b/httprunner/loader.py index 59b8203a..fc0fbd33 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -315,12 +315,70 @@ def get_module_item(module_mapping, item_type, item_name): ## testcase loader ############################################################################### +def _load_teststeps(test_block, project_mapping): + """ load teststeps with api/testcase references + + Args: + test_block (dict): test block content, maybe in 3 formats. + # api reference + { + "name": "add product to cart", + "api": "api_add_cart()", + "validate": [] + } + # testcase reference + { + "name": "add product to cart", + "suite": "create_and_check()", + "validate": [] + } + # define directly + { + "name": "checkout cart", + "request": {}, + "validate": [] + } + + Returns: + list: loaded teststeps list + + """ + def extend_api_definition(block): + ref_call = block["api"] + def_block = _get_block_by_name(ref_call, "def-api", project_mapping) + _extend_block(block, def_block) + + teststeps = [] + + # reference api + if "api" in test_block: + extend_api_definition(test_block) + teststeps.append(test_block) + + # reference testcase + elif "suite" in test_block: # TODO: replace suite with testcase + ref_call = test_block["suite"] + block = _get_block_by_name(ref_call, "def-testcase", project_mapping) + # TODO: bugfix lost block config variables + for teststep in block["teststeps"]: + if "api" in teststep: + extend_api_definition(teststep) + teststeps.append(teststep) + + # define directly + else: + teststeps.append(test_block) + + return teststeps + + def _load_testcase(raw_testcase, project_mapping): """ load testcase/testsuite with api/testcase references Args: raw_testcase (list): raw testcase content loaded from JSON/YAML file: [ + # config part { "config": { "name": "", @@ -328,26 +386,12 @@ def _load_testcase(raw_testcase, project_mapping): "request": {} } }, + # teststeps part { - "test": { - "name": "add product to cart", - "api": "api_add_cart()", - "validate": [] - } + "test": {...} }, { - "test": { - "name": "add product to cart", - "suite": "create_and_check()", - "validate": [] - } - }, - { - "test": { - "name": "checkout cart", - "request": {}, - "validate": [] - } + "test": {...} } ] project_mapping (dict): project_mapping @@ -378,30 +422,7 @@ def _load_testcase(raw_testcase, project_mapping): loaded_testcase["config"].update(test_block) elif key == "test": - - def extend_api_definition(block): - ref_call = block["api"] - def_block = _get_block_by_name(ref_call, "def-api", project_mapping) - _extend_block(block, def_block) - - # reference api - if "api" in test_block: - extend_api_definition(test_block) - loaded_testcase["teststeps"].append(test_block) - - # reference testcase - elif "suite" in test_block: # TODO: replace suite with testcase - ref_call = test_block["suite"] - block = _get_block_by_name(ref_call, "def-testcase", project_mapping) - # TODO: bugfix lost block config variables - for teststep in block["teststeps"]: - if "api" in teststep: - extend_api_definition(teststep) - loaded_testcase["teststeps"].append(teststep) - - # define directly - else: - loaded_testcase["teststeps"].append(test_block) + loaded_testcase["teststeps"].extend(_load_teststeps(test_block, project_mapping)) else: logger.log_warning( @@ -917,7 +938,7 @@ def load_tests(path, dot_env_path=None): "teststeps": [ # teststep data structure { - 'name': 'test step desc2', + 'name': 'test step desc1', 'variables': [], # optional 'extract': [], # optional 'validate': [], diff --git a/tests/test_loader.py b/tests/test_loader.py index ebeff85c..096b8511 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -283,7 +283,18 @@ class TestSuiteLoader(unittest.TestCase): def setUpClass(cls): cls.project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests")) - def test_load_test_file_testcase(self): + def test_load_teststeps(self): + test_block = { + "name": "setup and reset all.", + "suite": "setup_and_reset($device_sn)", + "output": ["token", "device_sn"] + } + teststeps = loader._load_teststeps(test_block, self.project_mapping) + self.assertEqual(len(teststeps), 2) + self.assertEqual(teststeps[0]["name"], "get token") + self.assertEqual(teststeps[1]["name"], "reset all users") + + def test_load_testcase(self): raw_testcase = loader.load_file("tests/testcases/smoketest.yml") testcase = loader._load_testcase(raw_testcase, self.project_mapping) self.assertEqual(testcase["config"]["name"], "smoketest") From 2d5ff04b46d02667de039705b1bc250aca9dc19e Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 29 Oct 2018 20:49:24 +0800 Subject: [PATCH 3/5] locusts support weight feature --- httprunner/__about__.py | 2 +- httprunner/__init__.py | 2 +- httprunner/api.py | 19 ---------- httprunner/context.py | 9 ++++- httprunner/loader.py | 47 ++++++++++++++++++++++++ httprunner/locusts.py | 3 -- httprunner/templates/locustfile_template | 44 ++++++++++++++++------ tests/data/demo_locust.yml | 34 +++++++++++++++++ tests/test_api.py | 14 +------ tests/test_loader.py | 9 +++++ 10 files changed, 133 insertions(+), 50 deletions(-) create mode 100644 tests/data/demo_locust.yml diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 82a265d0..211c90c2 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.5.14' +__version__ = '1.5.15' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/__init__.py b/httprunner/__init__.py index 48724cf5..6942cf6c 100644 --- a/httprunner/__init__.py +++ b/httprunner/__init__.py @@ -6,4 +6,4 @@ try: except ImportError: pass -from httprunner.api import HttpRunner, LocustRunner +from httprunner.api import HttpRunner diff --git a/httprunner/api.py b/httprunner/api.py index feb791a1..e5e5e97d 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -255,22 +255,3 @@ class HttpRunner(object): html_report_name, html_report_template ) - - -class LocustRunner(object): - - def __init__(self, locust_client): - self.runner = HttpRunner(http_client_session=locust_client) - - def run(self, path): - try: - self.runner.run(path) - except exceptions.MyBaseError as ex: - # TODO: refactor - from locust.events import request_failure - request_failure.fire( - request_type=test.testcase_dict.get("request", {}).get("method"), - name=test.testcase_dict.get("request", {}).get("url"), - response_time=0, - exception=ex - ) diff --git a/httprunner/context.py b/httprunner/context.py index 98e7ebce..7484a503 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -14,8 +14,13 @@ class Context(object): """ init Context with testcase variables and functions. """ # testcase level context - ## TESTCASE_SHARED_VARIABLES_MAPPING and TESTCASE_SHARED_FUNCTIONS_MAPPING will not change. - self.TESTCASE_SHARED_VARIABLES_MAPPING = variables or OrderedDict() + ## TESTCASE_SHARED_VARIABLES_MAPPING and TESTCASE_SHARED_FUNCTIONS_MAPPING are unchangeable. + if isinstance(variables, list): + self.TESTCASE_SHARED_VARIABLES_MAPPING = utils.convert_mappinglist_to_orderdict(variables) + else: + # dict + self.TESTCASE_SHARED_VARIABLES_MAPPING = variables or OrderedDict() + self.TESTCASE_SHARED_FUNCTIONS_MAPPING = functions or OrderedDict() # testcase level request, will not change diff --git a/httprunner/loader.py b/httprunner/loader.py index fc0fbd33..d8294c40 100644 --- a/httprunner/loader.py +++ b/httprunner/loader.py @@ -987,3 +987,50 @@ def load_tests(path, dot_env_path=None): testcases_list = [] return testcases_list + + +def load_locust_tests(path, dot_env_path=None): + """ load locust testcases + + Args: + path (str): testcase/testsuite file path. + dot_env_path (str): specified .env file path + + Returns: + dict: locust testcases with weight + { + "config": {...}, + "tests": [ + # weight 3 + [teststep11], + [teststep11], + [teststep11], + # weight 2 + [teststep21, teststep22], + [teststep21, teststep22] + ] + } + + """ + raw_testcase = load_file(path) + project_mapping = load_project_tests(path, dot_env_path) + + config = { + "refs": project_mapping + } + tests = [] + for item in raw_testcase: + key, test_block = item.popitem() + + if key == "config": + config.update(test_block) + elif key == "test": + teststeps = _load_teststeps(test_block, project_mapping) + weight = test_block.pop("weight", 1) + for _ in range(weight): + tests.append(teststeps) + + return { + "config": config, + "tests": tests + } diff --git a/httprunner/locusts.py b/httprunner/locusts.py index b1532229..3411bfd5 100644 --- a/httprunner/locusts.py +++ b/httprunner/locusts.py @@ -40,13 +40,10 @@ def gen_locustfile(testcase_file_path): "templates", "locustfile_template" ) - testcases = loader.load_tests(testcase_file_path) - host = testcases[0].get("config", {}).get("request", {}).get("base_url", "") with io.open(template_path, encoding='utf-8') as template: with io.open(locustfile_path, 'w', encoding='utf-8') as locustfile: template_content = template.read() - template_content = template_content.replace("$HOST", host) template_content = template_content.replace("$TESTCASE_FILE", testcase_file_path) locustfile.write(template_content) diff --git a/httprunner/templates/locustfile_template b/httprunner/templates/locustfile_template index 9b7c1924..84381083 100644 --- a/httprunner/templates/locustfile_template +++ b/httprunner/templates/locustfile_template @@ -1,24 +1,46 @@ -#coding: utf-8 +import logging +import random + import zmq +from httprunner.exceptions import MyBaseError, MyBaseFailure +from httprunner.loader import load_locust_tests +from httprunner.runner import Runner from locust import HttpLocust, TaskSet, task -from httprunner import LocustRunner -from httprunner.loader import load_tests +from locust.events import request_failure + +logging.getLogger().setLevel(logging.CRITICAL) +logging.getLogger('locust.main').setLevel(logging.INFO) +logging.getLogger('locust.runners').setLevel(logging.INFO) class WebPageTasks(TaskSet): def on_start(self): - self.test_runner = LocustRunner(self.client) - self.testcases = loader.load_tests(self.locust.file_path) + self.test_runner = Runner(self.locust.config, self.client) + self.testcases = load_locust_tests(self.locust.file_path) - @task - def test_specified_scenario(self): - self.test_runner.run(self.testcases) + @task(weight=1) + def test_any(self): + teststeps = random.choice(self.locust.tests) + for teststep in teststeps: + try: + self.test_runner.run_test(teststep) + except (MyBaseError, MyBaseFailure) as ex: + request_failure.fire( + request_type=teststep.get("request", {}).get("method"), + name=teststep.get("name"), + response_time=0, + exception=ex + ) class WebPageUser(HttpLocust): - host = "$HOST" task_set = WebPageTasks - min_wait = 1000 - max_wait = 5000 + min_wait = 10 + max_wait = 30 file_path = "$TESTCASE_FILE" + locust_tests = load_locust_tests(file_path) + config = locust_tests["config"] + tests = locust_tests["tests"] + + host = config.get('request', {}).get('base_url', '') diff --git a/tests/data/demo_locust.yml b/tests/data/demo_locust.yml new file mode 100644 index 00000000..8965689c --- /dev/null +++ b/tests/data/demo_locust.yml @@ -0,0 +1,34 @@ +- config: + name: basic test with httpbin + request: + base_url: https://httpbin.org/ + +- test: + name: index + weight: 5 + request: + url: / + method: GET + validate: + - eq: ["status_code", 200] + - contains: [content, "HTTP Request & Response Service"] + +- test: + name: headers + weight: 3 + request: + url: /headers + method: GET + validate: + - eq: ["status_code", 200] + - eq: [content.headers.Host, "httpbin.org"] + +- test: + name: user-agent + weight: 2 + request: + url: /user-agent + method: GET + validate: + - eq: ["status_code", 200] + - startswith: [content.user-agent, "python-requests"] diff --git a/tests/test_api.py b/tests/test_api.py index bd869938..7bf769a6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2,7 +2,7 @@ import os import shutil import time -from httprunner import HttpRunner, LocustRunner, api, loader, parser +from httprunner import HttpRunner, api, loader, parser from locust import HttpLocust from tests.api_server import HTTPBIN_SERVER from tests.base import ApiServerUnittest @@ -375,15 +375,3 @@ class TestHttpRunner(ApiServerUnittest): os.getcwd(), 'tests/httpbin/basic.yml') runner = HttpRunner().run(testcase_file_path) self.assertTrue(runner.summary["success"]) - - -class TestLocustRunner(ApiServerUnittest): - - def setUp(self): - WebPageUser = type('WebPageUser', (HttpLocust,), {}) - self.locust_client = WebPageUser.client - - def test_LocustRunner(self): - testcase_file = os.path.join(os.getcwd(), 'tests', 'httpbin', 'basic.yml') - locust_runner = LocustRunner(self.locust_client) - locust_runner.run(testcase_file) diff --git a/tests/test_loader.py b/tests/test_loader.py index 096b8511..5d0a2f08 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -553,3 +553,12 @@ class TestSuiteLoader(unittest.TestCase): self.assertIn("get_token", project_mapping["def-api"]) self.assertIn("setup_and_reset", project_mapping["def-testcase"]) self.assertEqual(project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH") + + def test_load_locust_tests(self): + path = os.path.join( + os.getcwd(), 'tests/data/demo_locust.yml') + locust_tests = loader.load_locust_tests(path) + self.assertEqual(locust_tests["config"]["refs"]["env"]["UserName"], "debugtalk") + self.assertEqual(len(locust_tests["tests"]), 10) + self.assertEqual(locust_tests["tests"][0][0]["name"], "index") + self.assertEqual(locust_tests["tests"][9][0]["name"], "user-agent") From 8aeb8d4dc2275eee0a5fc089e407e3c93e7118bd Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 31 Oct 2018 23:21:37 +0800 Subject: [PATCH 4/5] set logging level for locusts --- httprunner/cli.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/httprunner/cli.py b/httprunner/cli.py index b675ecd8..f0f1bda7 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -101,20 +101,34 @@ def main_hrun(): def main_locust(): """ Performance test with locust: parse command line options and run commands. """ - logger.setup_logger("INFO") - try: from httprunner import locusts except ImportError: msg = "Locust is not installed, install first and try again.\n" msg += "install command: pip install locustio" - logger.log_warning(msg) + print(msg) exit(1) sys.argv[0] = 'locust' if len(sys.argv) == 1: sys.argv.extend(["-h"]) + # set logging level + if "-L" in sys.argv: + loglevel_index = sys.argv.index('-L') + 1 + elif "--loglevel" in sys.argv: + loglevel_index = sys.argv.index('--loglevel') + 1 + else: + loglevel_index = None + + if loglevel_index and loglevel_index < len(sys.argv): + loglevel = sys.argv[loglevel_index] + else: + # default + loglevel = "INFO" + + logger.setup_logger(loglevel) + if sys.argv[1] in ["-h", "--help", "-V", "--version"]: locusts.main() sys.exit(0) From 8c4a745d97d5319c261254824ac4868ac61f1317 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 31 Oct 2018 23:30:08 +0800 Subject: [PATCH 5/5] add --locustfile argument --- httprunner/cli.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/httprunner/cli.py b/httprunner/cli.py index f0f1bda7..cfbdb0df 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -113,6 +113,10 @@ def main_locust(): if len(sys.argv) == 1: sys.argv.extend(["-h"]) + if sys.argv[1] in ["-h", "--help", "-V", "--version"]: + locusts.main() + sys.exit(0) + # set logging level if "-L" in sys.argv: loglevel_index = sys.argv.index('-L') + 1 @@ -129,15 +133,18 @@ def main_locust(): logger.setup_logger(loglevel) - if sys.argv[1] in ["-h", "--help", "-V", "--version"]: - locusts.main() - sys.exit(0) - + # get testcase file path try: - testcase_index = sys.argv.index('-f') + 1 - assert testcase_index < len(sys.argv) - except (ValueError, AssertionError): - logger.log_error("Testcase file is not specified, exit.") + if "-f" in sys.argv: + testcase_index = sys.argv.index('-f') + 1 + elif "--locustfile" in sys.argv: + testcase_index = sys.argv.index('--locustfile') + 1 + else: + testcase_index = None + + assert testcase_index and testcase_index < len(sys.argv) + except AssertionError: + print("Testcase file is not specified, exit.") sys.exit(1) testcase_file_path = sys.argv[testcase_index]