GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-06-11 10:49:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor testcase layer mechanism:

Browse Source 
1, autotest testsuite layer
2, performance test
This commit is contained in:
debugtalk
2018-12-10 15:57:06 +08:00
parent 4939f2cc0c
commit 7d20f95068
13 changed files with 677 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
httprunner/api.py
View File

@@ -69,7 +69,7 @@ class HttpRunner(object):
test_runner = runner.Runner(config, functions)
TestSequense = type('TestSequense', (unittest.TestCase,), {})
tests = testcase.get("tests", [])
tests = testcase.get("teststeps", [])
for index, test_dict in enumerate(tests):
for times_index in range(int(test_dict.get("times", 1))):
# suppose one testcase should not have more than 9999 steps,
@@ -80,7 +80,7 @@ class HttpRunner(object):
loaded_testcase = self.test_loader.loadTestsFromTestCase(TestSequense)
setattr(loaded_testcase, "config", config)
setattr(loaded_testcase, "tests", tests)
setattr(loaded_testcase, "teststeps", tests)
setattr(loaded_testcase, "runner", test_runner)
test_suite.addTest(loaded_testcase)
@@ -145,14 +145,14 @@ class HttpRunner(object):
"""
# parse tests
self.exception_stage = "parse tests"
parser.parse_tests(tests_mapping)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
if self.save_tests:
utils.dump_tests(tests_mapping, "parsed")
utils.dump_tests(parsed_tests_mapping, "parsed")
# add tests to test suite
self.exception_stage = "add tests to test suite"
test_suite = self._add_tests(tests_mapping)
test_suite = self._add_tests(parsed_tests_mapping)
# run test suite
self.exception_stage = "run test suite"
@@ -236,16 +236,16 @@ def prepare_locust_tests(path):
{
"functions": {},
"tests": []
"teststeps": []
}
"""
tests_mapping = loader.load_tests(path)
parser.parse_tests(tests_mapping)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
functions = tests_mapping.get("project_mapping", {}).get("functions", {})
testcase = tests_mapping["testcases"][0]
items = testcase.get("tests", [])
functions = parsed_tests_mapping.get("project_mapping", {}).get("functions", {})
testcase = parsed_tests_mapping["testcases"][0]
items = testcase.get("teststeps", [])
tests = []
for item in items:

280
httprunner/loader.py
View File

@@ -286,75 +286,95 @@ tests_def_mapping = {
"testcases": {}
}
def load_test(raw_testinfo):
""" load test with api/testcase/proc references
def __extend_with_api_ref(raw_testinfo):
""" extend with api reference
Raises:
exceptions.ApiNotFound: api not found
"""
api_name = raw_testinfo["api"]
# api maybe defined in two types:
# 1, individual file: each file is corresponding to one api definition
# 2, api sets file: one file contains a list of api definitions
if not os.path.isabs(api_name):
api_path = os.path.join(tests_def_mapping["PWD"], api_name)
if os.path.isfile(api_path):
# type 1: api is defined in individual file
api_name = api_path
try:
block = tests_def_mapping["api"][api_name]
# NOTICE: avoid project_mapping been changed during iteration.
raw_testinfo["api_def"] = utils.deepcopy_dict(block)
except KeyError:
raise exceptions.ApiNotFound("{} not found!".format(name))
def __extend_with_testcase_ref(raw_testinfo):
""" extend with testcase reference
"""
testcase_path = raw_testinfo["testcase"]
if testcase_path not in tests_def_mapping["testcases"]:
testcase_path = os.path.join(
project_mapping["PWD"],
testcase_path
)
testcase_dict = load_testcase(load_file(testcase_path))
tests_def_mapping[testcase_path] = testcase_dict
else:
testcase_dict = tests_def_mapping[testcase_path]
raw_testinfo["testcase_def"] = testcase_dict
def load_teststep(raw_testinfo):
""" load testcase step content.
teststep maybe defined directly, or reference api/testcase.
Args:
raw_testinfo (dict): test data, maybe in 3 formats.
# api reference
{
"name": "add product to cart",
"api": "api_add_cart",
"variables": [],
"api": "/path/to/api",
"variables": {},
"validate": [],
"extract": {}
}
# testcase reference
{
"name": "add product to cart",
"testcase": "create_and_check",
"variables": []
"testcase": "/path/to/testcase",
"variables": {}
}
# define directly
{
"name": "checkout cart",
"request": {},
"variables": [],
"variables": {},
"validate": [],
"extract": {}
}
Returns:
list: loaded tests list
Args:
raw_testinfo (dict): test info
dict: loaded teststep content
"""
# reference api
if "api" in raw_testinfo:
api_name = raw_testinfo["api"]
# api maybe defined in two types:
# 1, individual file: each file is corresponding to one api definition
# 2, api sets file: one file contains a list of api definitions
if not os.path.isabs(api_name):
api_path = os.path.join(tests_def_mapping["PWD"], api_name)
if os.path.isfile(api_path):
# type 1: api is defined in individual file
api_name = api_path
raw_testinfo["api_def"] = _get_api_definition(api_name)
__extend_with_api_ref(raw_testinfo)
# TODO: reference proc functions
elif "func" in raw_testinfo:
pass
# elif "func" in raw_testinfo:
# pass
# reference testcase
elif "testcase" in raw_testinfo:
testcase_path = raw_testinfo["testcase"]
if testcase_path not in tests_def_mapping["testcases"]:
testcase_path = os.path.join(
project_mapping["PWD"],
testcase_path
)
testcase_dict = load_testcase(load_file(testcase_path))
tests_def_mapping[testcase_path] = testcase_dict
else:
testcase_dict = tests_def_mapping[testcase_path]
raw_testinfo["testcase_def"] = testcase_dict
__extend_with_testcase_ref(raw_testinfo)
# define directly
else:
@@ -364,7 +384,7 @@ def load_test(raw_testinfo):
def load_testcase(raw_testcase):
""" load testcase/testsuite with api/testcase references
""" load testcase with api/testcase references.
Args:
raw_testcase (list): raw testcase content loaded from JSON/YAML file:
@@ -372,12 +392,11 @@ def load_testcase(raw_testcase):
# config part
{
"config": {
"name": "",
"def": "suite_order()",
"request": {}
"name": "XXXX",
"base_url": "https://debugtalk.com"
}
},
# tests part
# teststeps part
{
"test": {...}
},
@@ -389,9 +408,8 @@ def load_testcase(raw_testcase):
Returns:
dict: loaded testcase content
{
"name": "XYZ",
"config": {},
"tests": [test11, test12]
"teststeps": [test11, test12]
}
"""
@@ -399,20 +417,11 @@ def load_testcase(raw_testcase):
tests = []
for item in raw_testcase:
# TODO: add json schema validation
if not isinstance(item, dict) or len(item) != 1:
raise exceptions.FileFormatError("Testcase format error: {}".format(item))
key, test_block = item.popitem()
if not isinstance(test_block, dict):
raise exceptions.FileFormatError("Testcase format error: {}".format(item))
if key == "config":
config.update(test_block)
elif key == "test":
tests.append(load_test(test_block))
tests.append(load_teststep(test_block))
else:
logger.log_warning(
"unexpected block key: {}. block key should only be 'config' or 'test'.".format(key)
@@ -420,26 +429,113 @@ def load_testcase(raw_testcase):
return {
"config": config,
"tests": tests
"teststeps": tests
}
def _get_api_definition(name):
""" get api definition by name.
def load_testsuite(raw_testsuite):
""" load testsuite with testcase references.
Args:
raw_testsuite (dict): raw testsuite content loaded from JSON/YAML file:
{
"config": {
"name": "",
"request": {}
}
"testcases": {
"testcase1": {
"testcase": "/path/to/testcase",
"variables": {...},
"parameters": {...}
},
"testcase2": {}
}
}
Returns:
dict: expected api definition if found.
Raises:
exceptions.ApiNotFound: api not found
dict: loaded testsuite content
{
"config": {},
"testcases": [testcase1, testcase2]
}
"""
try:
block = tests_def_mapping["api"][name]
# NOTICE: avoid project_mapping been changed during iteration.
return utils.deepcopy_dict(block)
except KeyError:
raise exceptions.ApiNotFound("{} not found!".format(name))
testcases = raw_testsuite["testcases"]
for name, raw_testcase in testcases.items():
__extend_with_testcase_ref(raw_testcase)
raw_testcase.setdefault("name", name)
return raw_testsuite
def load_test_file(path):
""" load test file, file maybe testcase/testsuite/api
Args:
path (str): test file path
Returns:
dict: loaded test content
# api
{
"path": path,
"type": "api",
"name": "",
"request": {}
}
# testcase
{
"path": path,
"type": "testcase",
"config": {},
"teststeps": []
}
# testsuite
{
"path": path,
"type": "testsuite",
"config": {},
"testcases": {}
}
"""
raw_content = load_file(path)
loaded_content = None
if isinstance(raw_content, dict):
if "testcases" in raw_content:
# file_type: testsuite
# TODO: add json schema validation for testsuite
loaded_content = load_testsuite(raw_content)
loaded_content["path"] = path
loaded_content["type"] = "testsuite"
elif "request" in raw_content:
# file_type: api
# TODO: add json schema validation for api
loaded_content = raw_content
loaded_content["path"] = path
loaded_content["type"] = "api"
else:
# invalid format
logger.log_warning("Invalid test file format: {}".format(path))
elif isinstance(raw_content, list) and len(raw_content) > 0:
# file_type: testcase
# TODO: add json schema validation for testcase
loaded_content = load_testcase(raw_content)
loaded_content["path"] = path
loaded_content["type"] = "testcase"
else:
# invalid format
logger.log_warning("Invalid test file format: {}".format(path))
return loaded_content
def load_folder_content(folder_path):
@@ -623,7 +719,7 @@ def load_tests(path, dot_env_path=None):
"path": "testcase1_path",
"variables": [], # optional
},
"tests": [
"teststeps": [
# test data structure
{
'name': 'test desc1',
@@ -635,7 +731,17 @@ def load_tests(path, dot_env_path=None):
test_dict_2 # another test dict
]
},
testcase_dict_2 # another testcase dict
testcase_2_dict # another testcase dict
],
"testsuites": [
{ # testsuite data structure
"config": {},
"testcases": {
"testcase1": {},
"testcase2": {},
}
},
testsuite_2_dict
]
}
@@ -653,33 +759,23 @@ def load_tests(path, dot_env_path=None):
"project_mapping": project_mapping
}
def load_test_file(path):
raw_testcase = load_file(path)
try:
testcase = load_testcase(raw_testcase)
testcase["config"]["path"] = path
except exceptions.FileFormatError:
testcase = {}
return testcase
testcases_list = []
def __load_file_content(path):
loaded_content = load_test_file(path)
if not loaded_content:
pass
elif loaded_content["type"] == "testsuite":
tests_mapping.setdefault("testsuites", []).append(loaded_content)
elif loaded_content["type"] == "testcase":
tests_mapping.setdefault("testcases", []).append(loaded_content)
elif loaded_content["type"] == "api":
tests_mapping.setdefault("api", []).append(loaded_content)
if os.path.isdir(path):
files_list = load_folder_files(path)
for path in files_list:
testcase = load_test_file(path)
if not testcase:
continue
testcases_list.append(testcase)
__load_file_content(path)
elif os.path.isfile(path):
testcase = load_test_file(path)
if testcase:
testcases_list.append(testcase)
tests_mapping["testcases"] = testcases_list
__load_file_content(path)
return tests_mapping

286
httprunner/parser.py
View File

@@ -676,7 +676,7 @@ def _extend_with_testcase(test_dict, testcase_def_dict):
def __parse_config(config, project_mapping):
""" parse testcase config, include variables and name.
""" parse testcase/testsuite config, include variables and name.
"""
# get config variables
raw_config_variables = config.pop("variables", {})
@@ -684,31 +684,29 @@ def __parse_config(config, project_mapping):
override_variables = utils.deepcopy_dict(project_mapping.get("variables", {}))
functions = project_mapping.get("functions", {})
# override testcase config variables with passed in variables
# override config variables with passed in variables
raw_config_variables_mapping.update(override_variables)
# parse config variables
parsed_config_variables = {}
for key, value in raw_config_variables_mapping.items():
try:
parsed_value = parse_data(
value,
raw_config_variables_mapping,
functions
)
except exceptions.VariableNotFound:
pass
parsed_config_variables[key] = parsed_value
if key in override_variables:
# passed in
continue
else:
# config variables
try:
parsed_value = parse_data(
value,
override_variables,
functions
)
except exceptions.VariableNotFound:
pass
override_variables[key] = parsed_value
if override_variables:
config["variables"] = override_variables
if parsed_config_variables:
config["variables"] = parsed_config_variables
# parse config name
config["name"] = parse_data(
config.get("name", ""),
override_variables,
parsed_config_variables,
functions
)
@@ -716,17 +714,17 @@ def __parse_config(config, project_mapping):
if "base_url" in config:
config["base_url"] = parse_data(
config["base_url"],
override_variables,
parsed_config_variables,
functions
)
def __parse_tests(tests, config, project_mapping):
def __parse_testcase_tests(tests, config, project_mapping):
""" override tests with testcase config variables, base_url and verify.
test maybe nested testcase.
variables priority:
testsuite config > testsuite test > testcase config > testcase test > api
testcase config > testcase test > testcase_def config > testcase_def test > api
base_url/verify priority:
testcase test > testcase config > testsuite test > testsuite config > api
@@ -734,9 +732,7 @@ def __parse_tests(tests, config, project_mapping):
Args:
tests (list):
config (dict):
Returns:
list: overrided tests
project_mapping (dict):
"""
config_variables = config.pop("variables", {})
@@ -752,51 +748,34 @@ def __parse_tests(tests, config, project_mapping):
test_dict.setdefault("verify", config_verify)
# 1, testcase config => testcase tests
# override test_dict variables
test_dict["variables"] = utils.extend_variables(
test_dict.pop("variables", {}),
config_variables
)
# parse test_dict name
try:
test_dict["name"] = parse_data(
test_dict.pop("name", ""),
test_dict["variables"],
functions
)
except exceptions.VariableNotFound:
pass
if "testcase_def" in test_dict:
# test_dict is nested testcase
# 1, testsuite config => testsuite tests
# override test_dict variables
test_dict["variables"] = utils.extend_variables(
test_dict.pop("variables", {}),
config_variables
)
# parse test_dict name
try:
test_dict["name"] = parse_data(
test_dict.pop("name", ""),
test_dict["variables"],
functions
)
except exceptions.VariableNotFound:
pass
# 2, testsuite test_dict => testcase config
# 2, testcase test_dict => testcase_def config
testcase_def = test_dict.pop("testcase_def")
_extend_with_testcase(test_dict, testcase_def)
# 3, testcase config => testcase test_dict
# 3, testcase_def config => testcase_def test_dict
_parse_testcase(test_dict, project_mapping)
else:
# 1, config => tests
# override test_dict variables
test_dict["variables"] = utils.extend_variables(
test_dict.pop("variables", {}),
config_variables
)
# parse test_dict name
try:
test_dict["name"] = parse_data(
test_dict.pop("name", ""),
test_dict["variables"],
functions
)
except exceptions.VariableNotFound:
pass
if "api_def" in test_dict:
# test_dict has API reference
# 2, test_dict => api
@@ -804,11 +783,14 @@ def __parse_tests(tests, config, project_mapping):
_extend_with_api(test_dict, api_def_dict)
if test_dict.get("base_url"):
# parse base_url
base_url = parse_data(
test_dict.pop("base_url"),
test_dict["variables"],
functions
)
# build path with base_url
try:
request_url = parse_data(
test_dict["request"]["url"],
@@ -817,7 +799,7 @@ def __parse_tests(tests, config, project_mapping):
)
except exceptions.VariableNotFound:
""" variable in current url maybe extracted from former api
"tests": [
"teststeps": [
{
"request": {},
"extract": {
@@ -840,12 +822,125 @@ def __parse_tests(tests, config, project_mapping):
def _parse_testcase(testcase, project_mapping):
""" parse testcase
Args:
testcase (dict):
{
"config": {},
"teststeps": []
}
"""
testcase.setdefault("config", {})
__parse_config(testcase["config"], project_mapping)
__parse_tests(testcase["tests"], testcase["config"], project_mapping)
__parse_testcase_tests(testcase["teststeps"], testcase["config"], project_mapping)
def __get_parsed_testsuite_testcases(testcases, testsuite_config, project_mapping):
""" override testscases with testsuite config variables, base_url and verify.
variables priority:
testsuite config > testcase config > testcase_def config > testcase_def tests > api
Args:
testcases (dict):
{
"testcase1 name": {
"testcase": "testcases/create_and_check.yml",
"weight": 2,
"variables": {
"uid": 1000
},
"parameters": {
"uid": [100, 101, 102]
},
"testcase_def": {
"config": {},
"teststeps": []
}
},
"testcase2 name": {}
}
testsuite_config (dict):
{
"name": "testsuite name",
"variables": {
"device_sn": "${gen_random_string(15)}"
},
"base_url": "http://127.0.0.1:5000"
}
project_mapping (dict):
{
"env": {},
"functions": {}
}
"""
testsuite_config_variables = testsuite_config.get("variables", {})
functions = project_mapping.get("functions", {})
parsed_testcase_list = []
for testcase_name, testcase in testcases.items():
# TODO: add parameterize
# parameters = testcase.get("parameters")
parsed_testcase = testcase.pop("testcase_def")
parsed_testcase.setdefault("config", {})
parsed_testcase["path"] = testcase["testcase"]
parsed_testcase["config"]["name"] = testcase_name
# 1, testsuite config => testcase config
# override test_dict variables
testcase_config_variables = utils.extend_variables(
testcase.pop("variables", {}),
testsuite_config_variables
)
# 2, testcase config > testcase_def config
# override testcase_def config variables
parsed_testcase_config_variables = utils.extend_variables(
parsed_testcase["config"].pop("variables", {}),
testcase_config_variables
)
# parse config variables
parsed_config_variables = {}
for key, value in parsed_testcase_config_variables.items():
try:
parsed_value = parse_data(
value,
parsed_testcase_config_variables,
functions
)
except exceptions.VariableNotFound:
pass
parsed_config_variables[key] = parsed_value
if parsed_config_variables:
parsed_testcase["config"]["variables"] = parsed_config_variables
_parse_testcase(parsed_testcase, project_mapping)
parsed_testcase_list.append(parsed_testcase)
return parsed_testcase_list
def _parse_testsuite(testsuite, project_mapping):
testsuite.setdefault("config", {})
__parse_config(testsuite["config"], project_mapping)
parsed_testcase_list = __get_parsed_testsuite_testcases(
testsuite["testcases"],
testsuite["config"],
project_mapping
)
return parsed_testcase_list
def parse_tests(tests_mapping):
""" parse testcases configs, including variables/name/request.
""" parse tests and load to parsed testcases
tests include api, testcases and testsuites.
Args:
tests_mapping (dict): project info and testcases list.
@@ -857,14 +952,34 @@ def parse_tests(tests_mapping):
"variables": {}, # optional, priority 1
"env": {}
},
"testsuites": [
{ # testsuite data structure
"config": {},
"testcases": {
"testcase1 name": {
"variables": {
"uid": 1000
},
"parameters": {
"uid": [100, 101, 102]
},
"testcase_def": {
"config": {},
"teststeps": []
}
},
"testcase2 name": {}
}
}
],
"testcases": [
{ # testcase data structure
"config": {
"name": "desc1",
"path": "testcase1_path",
"variables": [], # optional, priority 2
"variables": {}, # optional, priority 2
},
"tests": [
"teststeps": [
# test data structure
{
'name': 'test step desc1',
@@ -880,12 +995,41 @@ def parse_tests(tests_mapping):
]
},
testcase_dict_2 # another testcase dict
]
],
"api": {
"variables": {},
"request": {}
}
}
"""
project_mapping = tests_mapping.get("project_mapping", {})
parsed_tests_mapping = {
"project_mapping": project_mapping,
"testcases": []
}
for testcase in tests_mapping["testcases"]:
testcase.setdefault("config", {})
_parse_testcase(testcase, project_mapping)
for test_type in tests_mapping:
if test_type == "testsuites":
# load testcases of testsuite
testsuites = tests_mapping["testsuites"]
for testsuite in testsuites:
parsed_testcases = _parse_testsuite(testsuite, project_mapping)
for parsed_testcase in parsed_testcases:
parsed_tests_mapping["testcases"].append(parsed_testcase)
elif test_type == "testcases":
for testcase in tests_mapping["testcases"]:
_parse_testcase(testcase, project_mapping)
parsed_tests_mapping["testcases"].append(testcase)
elif test_type == "api":
# encapsulate api as a testcase
testcase = {
"teststeps": tests_mapping["api"]
}
_parse_testcase(testcase, project_mapping)
parsed_tests_mapping["testcases"].append(testcase)
return parsed_tests_mapping

4
httprunner/runner.py
View File

@@ -282,7 +282,7 @@ class Runner(object):
http_client_session = self.http_client_session.__class__(base_url)
test_runner = Runner(config, self.functions, http_client_session)
tests = testcase_dict.get("tests", [])
tests = testcase_dict.get("teststeps", [])
try:
for index, test_dict in enumerate(tests):
@@ -317,7 +317,7 @@ class Runner(object):
# nested testcase
{
"config": {...},
"tests": [
"teststeps": [
{...},
{...}
]

8
httprunner/utils.py
View File

@@ -689,11 +689,17 @@ def dump_tests(tests_mapping, tag_name):
tests_to_dump["project_mapping"][key] = project_mapping[key]
continue
# remove functions in order to dump
if project_mapping["functions"]:
debugtalk_py_path = os.path.join(pwd_dir_path, "debugtalk.py")
tests_to_dump["project_mapping"]["debugtalk.py"] = debugtalk_py_path
tests_to_dump["testcases"] = tests_mapping["testcases"]
if "api" in tests_mapping:
tests_to_dump["api"] = tests_mapping["api"]
elif "testcases" in tests_mapping:
tests_to_dump["testcases"] = tests_mapping["testcases"]
elif "testsuites" in tests_mapping:
tests_to_dump["testsuites"] = tests_mapping["testsuites"]
dump_json_file(tests_to_dump, pwd_dir_path, dump_file_name)

8
httprunner/validator.py
View File

@@ -19,7 +19,7 @@ def is_testcase(data_structure):
"variables": [], # optional
"request": {} # optional
},
"tests": [
"teststeps": [
test_dict1,
{ # test_dict2
'name': 'test step desc2',
@@ -40,10 +40,10 @@ def is_testcase(data_structure):
if not isinstance(data_structure, dict):
return False
if "tests" not in data_structure:
if "teststeps" not in data_structure:
return False
if not isinstance(data_structure["tests"], list):
if not isinstance(data_structure["teststeps"], list):
return False
return True
@@ -67,7 +67,7 @@ def is_testcases(data_structure):
"path": "testcase1_path",
"variables": [], # optional
},
"tests": [
"teststeps": [
# test data structure
{
'name': 'test step desc1',