HttpRunner 2.0 is comming!

1, new design for testcase format;
2, refactor testcase layer mechanism.
This commit is contained in:
debugtalk
2018-11-22 19:20:30 +08:00
parent 82b527d8b2
commit 4099ade49d
31 changed files with 1500 additions and 1303 deletions

View File

@@ -273,15 +273,21 @@ def load_debugtalk_functions():
## testcase loader
###############################################################################
def _load_teststeps(test_block, project_mapping):
""" load teststeps with api/testcase references
project_mapping = {}
tests_def_mapping = {
"api": {},
"testcases": {}
}
def load_teststep(raw_stepinfo):
""" load teststep with api/testcase/proc references
Args:
test_block (dict): test block content, maybe in 3 formats.
raw_stepinfo (dict): teststep data, maybe in 3 formats.
# api reference
{
"name": "add product to cart",
"api": "api_add_cart()",
"api": "api_add_cart",
"variables": [],
"validate": [],
"extract": []
@@ -289,7 +295,7 @@ def _load_teststeps(test_block, project_mapping):
# testcase reference
{
"name": "add product to cart",
"suite": "create_and_check()",
"testcase": "create_and_check",
"variables": []
}
# define directly
@@ -304,33 +310,43 @@ def _load_teststeps(test_block, project_mapping):
Returns:
list: loaded teststeps list
"""
teststeps = []
Args:
raw_stepinfo (dict): teststep info
"""
# reference api
if "api" in test_block:
ref_call = test_block.pop("api")
def_block = _get_block_by_name(ref_call, "def-api", project_mapping)
extended_block = _extend_block(test_block, def_block)
teststeps.append(extended_block)
if "api" in raw_stepinfo:
api_name = raw_stepinfo["api"]
raw_stepinfo["api_def"] = _get_api_definition(api_name)
# TODO: reference proc functions
elif "func" in raw_stepinfo:
pass
# reference testcase
elif "suite" in test_block: # TODO: replace suite with testcase
ref_call = test_block.pop("suite")
def_block = _get_block_by_name(ref_call, "def-testcase", project_mapping)
# TODO: bugfix lost block config variables
for teststep in def_block["teststeps"]:
_teststeps = _load_teststeps(teststep, project_mapping)
teststeps.extend(_teststeps)
elif "testcase" in raw_stepinfo:
testcase_path = raw_stepinfo["testcase"]
if testcase_path not in tests_def_mapping["testcases"]:
testcase_path = os.path.join(
project_mapping["PWD"],
testcase_path
)
testcase_dict = load_testcase(load_file(testcase_path))
tests_def_mapping[testcase_path] = testcase_dict
else:
testcase_dict = tests_def_mapping[testcase_path]
raw_stepinfo["testcase_def"] = testcase_dict
# define directly
else:
teststeps.append(test_block)
pass
return teststeps
return raw_stepinfo
def _load_testcase(raw_testcase, project_mapping):
def load_testcase(raw_testcase):
""" load testcase/testsuite with api/testcase references
Args:
@@ -352,20 +368,18 @@ def _load_testcase(raw_testcase, project_mapping):
"test": {...}
}
]
project_mapping (dict): project_mapping
Returns:
dict: loaded testcase content
{
"name": "XYZ",
"config": {},
"teststeps": [teststep11, teststep12]
}
"""
loaded_testcase = {
"config": {},
"teststeps": []
}
config = {}
teststeps = []
for item in raw_testcase:
# TODO: add json schema validation
@@ -377,290 +391,38 @@ def _load_testcase(raw_testcase, project_mapping):
raise exceptions.FileFormatError("Testcase format error: {}".format(item))
if key == "config":
loaded_testcase["config"].update(test_block)
config.update(test_block)
elif key == "test":
loaded_testcase["teststeps"].extend(_load_teststeps(test_block, project_mapping))
teststeps.append(load_teststep(test_block))
else:
logger.log_warning(
"unexpected block key: {}. block key should only be 'config' or 'test'.".format(key)
)
return loaded_testcase
return {
"config": config,
"teststeps": teststeps
}
def _get_block_by_name(ref_call, ref_type, project_mapping):
""" get test content by reference name.
Args:
ref_call (str): call function.
e.g. api_v1_Account_Login_POST($UserName, $Password)
ref_type (enum): "def-api" or "def-testcase"
project_mapping (dict): project_mapping
def _get_api_definition(name):
""" get api definition by name.
Returns:
dict: api/testcase definition.
Raises:
exceptions.ParamsError: call args number is not equal to defined args number.
"""
function_meta = parser.parse_function(ref_call)
func_name = function_meta["func_name"]
call_args = function_meta["args"]
block = _get_test_definition(func_name, ref_type, project_mapping)
def_args = block.get("function_meta", {}).get("args", [])
if len(call_args) != len(def_args):
err_msg = "{}: call args number is not equal to defined args number!\n".format(func_name)
err_msg += "defined args: {}\n".format(def_args)
err_msg += "reference args: {}".format(call_args)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
args_mapping = {}
for index, item in enumerate(def_args):
if call_args[index] == item:
continue
args_mapping[item] = call_args[index]
if args_mapping:
block = parser.substitute_variables(block, args_mapping)
return block
def _get_test_definition(name, ref_type, project_mapping):
""" get expected api or testcase.
Args:
name (str): api or testcase name
ref_type (enum): "def-api" or "def-testcase"
project_mapping (dict): project_mapping
Returns:
dict: expected api/testcase info if found.
dict: expected api definition if found.
Raises:
exceptions.ApiNotFound: api not found
exceptions.TestcaseNotFound: testcase not found
"""
block = project_mapping.get(ref_type, {}).get(name)
# NOTICE: avoid project_mapping been changed during iteration.
block = copy.deepcopy(block)
if not block:
err_msg = "{} not found!".format(name)
if ref_type == "def-api":
raise exceptions.ApiNotFound(err_msg)
else:
# ref_type == "def-testcase":
raise exceptions.TestcaseNotFound(err_msg)
return block
def _extend_block(ref_block, def_block):
""" extend ref_block with def_block, ref_block will merge and override def_block.
Args:
def_block (dict): api definition dict.
ref_block (dict): reference block
Returns:
dict: extended reference block.
Examples:
>>> def_block = {
"name": "get token 1",
"request": {...},
"validate": [{'eq': ['status_code', 200]}]
}
>>> ref_block = {
"name": "get token 2",
"extract": [{"token": "content.token"}],
"validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]
}
>>> _extend_block(def_block, ref_block)
{
"name": "get token 2",
"request": {...},
"extract": [{"token": "content.token"}],
"validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]
}
"""
extended_block = {}
# override name
extended_block["name"] = ref_block.pop("name", None) or def_block.pop("name", "")
# override variables
def_variables = def_block.pop("variables", [])
ref_variables = ref_block.pop("variables", [])
extended_block["variables"] = _extend_variables(
def_variables,
ref_variables
)
# merge & override validators
def_validators = def_block.pop("validate", [])
ref_validators = ref_block.pop("validate", [])
extended_block["validate"] = _extend_validators(
def_validators,
ref_validators
)
# merge & override extractors
def_extrators = def_block.pop("extract", [])
ref_extractors = ref_block.pop("extract", [])
extended_block["extract"] = _extend_variables(
def_extrators,
ref_extractors
)
# TODO: merge & override request
def_request = def_block.pop("request", {})
ref_request = ref_block.pop("request", {})
extended_block["request"] = def_request
# merge & override setup_hooks
def_setup_hooks = def_block.pop("setup_hooks", [])
ref_setup_hooks = ref_block.pop("setup_hooks", [])
extended_block["setup_hooks"] = list(set(def_setup_hooks + ref_setup_hooks))
# merge & override teardown_hooks
def_teardown_hooks = def_block.pop("teardown_hooks", [])
ref_teardown_hooks = ref_block.pop("teardown_hooks", [])
extended_block["teardown_hooks"] = list(set(def_teardown_hooks + ref_teardown_hooks))
# TODO: extend with other ref block items, e.g. times
# extended_block.update(def_block)
extended_block.update(ref_block)
return extended_block
def _convert_validators_to_mapping(validators):
""" convert validators list to mapping.
Args:
validators (list): validators in list
Returns:
dict: validators mapping, use (check, comparator) as key.
Examples:
>>> validators = [
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": {"b": 1}, "expect": 200, "comparator": "eq"}
]
>>> _convert_validators_to_mapping(validators)
{
("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
}
"""
validators_mapping = {}
for validator in validators:
validator = parser.parse_validator(validator)
if not isinstance(validator["check"], collections.Hashable):
check = json.dumps(validator["check"])
else:
check = validator["check"]
key = (check, validator["comparator"])
validators_mapping[key] = validator
return validators_mapping
def _extend_validators(def_validators, ref_validators):
""" extend ref_validators with def_validators.
ref_validators will merge and override def_validators.
Args:
def_validators (list):
ref_validators (list):
Returns:
list: extended validators
Examples:
>>> def_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}]
>>> ref_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}]
>>> _extend_validators(def_validators, ref_validators)
[
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": "s2", "expect": 16, "comparator": "len_eq"},
{"check": "s3", "expect": 12, "comparator": "len_eq"}
]
"""
if not def_validators:
return ref_validators
elif not ref_validators:
return def_validators
else:
def_validators_mapping = _convert_validators_to_mapping(def_validators)
ref_validators_mapping = _convert_validators_to_mapping(ref_validators)
def_validators_mapping.update(ref_validators_mapping)
return list(def_validators_mapping.values())
def _extend_variables(def_variables, ref_variables):
""" extend ref_variables with def_variables.
ref_variables will merge and override def_variables.
Args:
def_variables (list):
ref_variables (list):
Returns:
list: extended variables
Examples:
>>> def_variables = [{"var1": "val1"}, {"var2": "val2"}]
>>> ref_variables = [{"var1": "val111"}, {"var3": "val3"}]
>>> _extend_variables(def_variables, ref_variables)
[
{"var1": "val111"},
{"var2": "val2"},
{"var3": "val3"}
]
"""
if not def_variables:
return ref_variables
elif not ref_variables:
return def_variables
else:
extended_variables_dict = OrderedDict()
for def_variable in def_variables:
var_name = list(def_variable.keys())[0]
extended_variables_dict[var_name] = def_variable[var_name]
for ref_variable in ref_variables:
if not ref_variable:
continue
var_name = list(ref_variable.keys())[0]
extended_variables_dict[var_name] = ref_variable[var_name]
extended_variables = []
for key, value in extended_variables_dict.items():
extended_variables.append({key: value})
return extended_variables
try:
block = tests_def_mapping["api"][name]
# NOTICE: avoid project_mapping been changed during iteration.
return utils.deepcopy_dict(block)
except KeyError:
raise exceptions.ApiNotFound("{} not found!".format(name))
def load_folder_content(folder_path):
@@ -737,99 +499,16 @@ def load_api_folder(api_folder_path):
for api_item in api_items:
key, api_dict = api_item.popitem()
# TODO: replace def with api file path
api_def = api_dict.pop("def")
function_meta = parser.parse_function(api_def)
func_name = function_meta["func_name"]
# TODO: replace id with api file path
api_id = api_dict.get("id")
if api_id in api_definition_mapping:
logger.log_warning("API definition duplicated: {}".format(api_id))
if func_name in api_definition_mapping:
logger.log_warning("API definition duplicated: {}".format(func_name))
api_dict["function_meta"] = function_meta
api_definition_mapping[func_name] = api_dict
api_definition_mapping[api_id] = api_dict
return api_definition_mapping
def load_test_folder(test_folder_path):
""" load testcases definitions from folder.
Args:
test_folder_path (str): testcases files folder.
testcase file should be in the following format:
[
{
"config": {
"def": "create_and_check",
"request": {},
"validate": []
}
},
{
"test": {
"api": "get_user",
"validate": []
}
}
]
Returns:
dict: testcases definition mapping.
{
"create_and_check": [
{"config": {}},
{"test": {}},
{"test": {}}
],
"tests/testcases/create_and_get.yml": [
{"config": {}},
{"test": {}},
{"test": {}}
]
}
"""
test_definition_mapping = {}
test_items_mapping = load_folder_content(test_folder_path)
for test_file_path, items in test_items_mapping.items():
# TODO: add JSON schema validation
testcase = {
"config": {},
"teststeps": []
}
for item in items:
key, block = item.popitem()
if key == "config":
testcase["config"].update(block)
if "def" not in block:
test_definition_mapping[test_file_path] = testcase
continue
# TODO: replace def with testcase file path
testcase_def = block.pop("def")
function_meta = parser.parse_function(testcase_def)
func_name = function_meta["func_name"]
if func_name in test_definition_mapping:
logger.log_warning("API definition duplicated: {}".format(func_name))
testcase["function_meta"] = function_meta
test_definition_mapping[func_name] = testcase
else:
# key == "test":
### TODO: extend suite with api
testcase["teststeps"].append(block)
return test_definition_mapping
def load_debugtalk_py(start_path):
""" locate debugtalk.py file and returns PWD and debugtalk.py functions.
@@ -878,22 +557,17 @@ def load_project_tests(test_path, dot_env_path=None):
dict: project loaded api/testcases definitions, environments and debugtalk.py functions.
"""
project_mapping = {}
# locate PWD and load debugtalk.py functions
project_working_directory, debugtalk_functions = load_debugtalk_py(test_path)
project_mapping["PWD"] = project_working_directory
project_mapping["functions"] = debugtalk_functions
# load .env
dot_env_path = dot_env_path or os.path.join(project_working_directory, ".env")
project_mapping["env"] = load_dot_env_file(dot_env_path)
# load api/testcases
project_mapping["def-api"] = load_api_folder(os.path.join(project_working_directory, "api"))
# TODO: replace suite with testcases
project_mapping["def-testcase"] = load_test_folder(os.path.join(project_working_directory, "suite"))
return project_mapping
# load api
tests_def_mapping["api"] = load_api_folder(os.path.join(project_working_directory, "api"))
def load_tests(path, dot_env_path=None):
@@ -901,54 +575,44 @@ def load_tests(path, dot_env_path=None):
Args:
path (str/list): testcase file/foler path.
path could be in several types:
path could be in 2 types:
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
dot_env_path (str): specified .env file path
Returns:
list: testcases list, each testcase is corresponding to a file
[
{ # testcase data structure
"config": {
"name": "desc1",
"path": "testcase1_path",
"variables": [], # optional
"request": {}, # optional
dict: tests mapping, include project_mapping and testcases.
each testcase is corresponding to a file.
{
"project_mapping": {
"PWD": "XXXXX",
"functions": {},
"env": {},
"def-api": {},
"def-testcase": {}
"env": {}
},
"teststeps": [
# teststep data structure
{
'name': 'test step desc1',
'variables': [], # optional
'extract': [], # optional
'validate': [],
'request': {},
'function_meta': {}
"testcases": [
{ # testcase data structure
"config": {
"name": "desc1",
"path": "testcase1_path",
"variables": [], # optional
},
"teststeps": [
# teststep data structure
{
'name': 'test step desc1',
'variables': [], # optional
'extract': [], # optional
'validate': [],
'request': {}
},
teststep2 # another teststep dict
]
},
teststep2 # another teststep dict
testcase_dict_2 # another testcase dict
]
},
testcase_dict_2 # another testcase dict
]
}
"""
if isinstance(path, (list, set)):
testcases_list = []
for file_path in set(path):
testcases = load_tests(file_path, dot_env_path)
if not testcases:
continue
testcases_list.extend(testcases)
return testcases_list
if not os.path.exists(path):
err_msg = "path not exist: {}".format(path)
logger.log_error(err_msg)
@@ -957,22 +621,41 @@ def load_tests(path, dot_env_path=None):
if not os.path.isabs(path):
path = os.path.join(os.getcwd(), path)
load_project_tests(path, dot_env_path)
tests_mapping = {
"project_mapping": project_mapping
}
def load_test_file(path):
raw_testcase = load_file(path)
try:
testcase = load_testcase(raw_testcase)
testcase["config"]["path"] = path
except exceptions.FileFormatError:
testcase = {}
return testcase
testcases_list = []
if os.path.isdir(path):
files_list = load_folder_files(path)
testcases_list = load_tests(files_list, dot_env_path)
for path in files_list:
testcase = load_test_file(path)
if not testcase:
continue
testcases_list.append(testcase)
elif os.path.isfile(path):
try:
raw_testcase = load_file(path)
project_mapping = load_project_tests(path, dot_env_path)
testcase = _load_testcase(raw_testcase, project_mapping)
testcase["config"]["path"] = path
testcase["config"].update(project_mapping)
testcases_list = [testcase]
except exceptions.FileFormatError:
testcases_list = []
return testcases_list
testcase = load_test_file(path)
if testcase:
testcases_list.append(testcase)
tests_mapping["testcases"] = testcases_list
return tests_mapping
def load_locust_tests(path, dot_env_path=None):
@@ -999,7 +682,7 @@ def load_locust_tests(path, dot_env_path=None):
"""
raw_testcase = load_file(path)
project_mapping = load_project_tests(path, dot_env_path)
load_project_tests(path, dot_env_path)
config = {}
tests = []
@@ -1009,10 +692,10 @@ def load_locust_tests(path, dot_env_path=None):
if key == "config":
config.update(test_block)
elif key == "test":
teststeps = _load_teststeps(test_block, project_mapping)
weight = test_block.pop("weight", 1)
teststep = load_teststep(test_block)
weight = test_block.get("weight", 1)
for _ in range(weight):
tests.append(teststeps)
tests.append(teststep)
# parse config variables
raw_config_variables = config.get("variables", [])