GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-07-12 16:01:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor loader: load testcases

Browse Source
This commit is contained in:
httprunner
2018-08-12 01:25:28 +08:00
parent e02aefa122
commit b665ad79ef
8 changed files with 355 additions and 374 deletions
Download Patch File Download Diff File Expand all files Collapse all files

505
httprunner/loader.py
View File

@@ -13,19 +13,19 @@ from httprunner.compat import OrderedDict
project_mapping = { project_mapping = {
"debugtalk": {}, "debugtalk": {},
"env": {}, "env": {},
"tests": { "def-api": {},
"api": {}, "def-testcase": {}
"testcases": {}
}
} }
""" dict: save project loaded api/testcases, environments and debugtalk.py module. """ dict: save project loaded api/testcases definitions, environments and debugtalk.py module.
""" """
testcases_cache_mapping = {}
############################################################################### ###############################################################################
## file loader ## file loader
############################################################################### ###############################################################################
def _check_format(file_path, content): def _check_format(file_path, content):
""" check testcase format if valid """ check testcase format if valid
""" """
@@ -367,84 +367,12 @@ def get_module_item(module_mapping, item_type, item_name):
## testcase loader ## testcase loader
############################################################################### ###############################################################################
overall_def_dict = {
"api": {},
"suite": {}
}
testcases_cache_mapping = {}
def _load_test_dependencies():
""" load all api and suite definitions.
default api folder is "$CWD/tests/api/".
default suite folder is "$CWD/tests/suite/".
"""
# TODO: cache api and suite loading
# load api definitions
api_def_folder = os.path.join(os.getcwd(), "tests", "api")
for test_file in load_folder_files(api_def_folder):
_load_api_file(test_file)
# load suite definitions
suite_def_folder = os.path.join(os.getcwd(), "tests", "suite")
for suite_file in load_folder_files(suite_def_folder):
suite = _load_test_file(suite_file)
if "def" not in suite["config"]:
raise exceptions.ParamsError("def missed in suite file: {}!".format(suite_file))
call_func = suite["config"]["def"]
function_meta = parser.parse_function(call_func)
suite["function_meta"] = function_meta
overall_def_dict["suite"][function_meta["func_name"]] = suite
def _load_api_file(file_path):
""" load api definition from file and store in overall_def_dict["api"]
api file should be in format below:
[
{
"api": {
"def": "api_login",
"request": {},
"validate": []
}
},
{
"api": {
"def": "api_logout",
"request": {},
"validate": []
}
}
]
"""
api_items = load_file(file_path)
if not isinstance(api_items, list):
raise exceptions.FileFormatError("API format error: {}".format(file_path))
for api_item in api_items:
if not isinstance(api_item, dict) or len(api_item) != 1:
raise exceptions.FileFormatError("API format error: {}".format(file_path))
key, api_dict = api_item.popitem()
if key != "api" or not isinstance(api_dict, dict) or "def" not in api_dict:
raise exceptions.FileFormatError("API format error: {}".format(file_path))
api_def = api_dict.pop("def")
function_meta = parser.parse_function(api_def)
func_name = function_meta["func_name"]
if func_name in overall_def_dict["api"]:
logger.log_warning("API definition duplicated: {}".format(func_name))
api_dict["function_meta"] = function_meta
overall_def_dict["api"][func_name] = api_dict
def _load_test_file(file_path): def _load_test_file(file_path):
""" load testcase file or testsuite file """ load testcase file or testsuite file
@param file_path: absolute valid file path
file_path should be in format below: Args:
file_path (str): absolute valid file path. file_path should be in the following format:
[ [
{ {
"config": { "config": {
@@ -460,6 +388,13 @@ def _load_test_file(file_path):
"validate": [] "validate": []
} }
}, },
{
"test": {
"name": "add product to cart",
"suite": "create_and_check()",
"validate": []
}
},
{ {
"test": { "test": {
"name": "checkout cart", "name": "checkout cart",
@@ -468,19 +403,24 @@ def _load_test_file(file_path):
} }
} }
] ]
@return testset dict
{ Returns:
"config": {}, dict: testcase dict
"testcases": [testcase11, testcase12] {
} "config": {},
"teststeps": [teststep11, teststep12]
}
""" """
testset = { testcase = {
"config": { "config": {
"path": file_path "path": file_path
}, },
"testcases": [] # TODO: rename to tests "teststeps": []
} }
for item in load_file(file_path): for item in load_file(file_path):
# TODO: add json schema validation
if not isinstance(item, dict) or len(item) != 1: if not isinstance(item, dict) or len(item) != 1:
raise exceptions.FileFormatError("Testcase format error: {}".format(file_path)) raise exceptions.FileFormatError("Testcase format error: {}".format(file_path))
@@ -489,43 +429,69 @@ def _load_test_file(file_path):
raise exceptions.FileFormatError("Testcase format error: {}".format(file_path)) raise exceptions.FileFormatError("Testcase format error: {}".format(file_path))
if key == "config": if key == "config":
testset["config"].update(test_block) testcase["config"].update(test_block)
elif key == "test": elif key == "test":
def extend_api_definition(block):
ref_call = block["api"]
def_block = _get_block_by_name(ref_call, "def-api")
_extend_block(block, def_block)
# reference api
if "api" in test_block: if "api" in test_block:
ref_call = test_block["api"] extend_api_definition(test_block)
def_block = _get_block_by_name(ref_call, "api") testcase["teststeps"].append(test_block)
_override_block(def_block, test_block)
testset["testcases"].append(test_block) # reference testcase
elif "suite" in test_block: elif "suite" in test_block: # TODO: replace suite with testcase
ref_call = test_block["suite"] ref_call = test_block["suite"]
block = _get_block_by_name(ref_call, "suite") block = _get_block_by_name(ref_call, "def-testcase")
testset["testcases"].extend(block["testcases"]) # TODO: bugfix lost block config variables
for teststep in block["teststeps"]:
if "api" in teststep:
extend_api_definition(teststep)
testcase["teststeps"].append(teststep)
# define directly
else: else:
testset["testcases"].append(test_block) testcase["teststeps"].append(test_block)
else: else:
logger.log_warning( logger.log_warning(
"unexpected block key: {}. block key should only be 'config' or 'test'.".format(key) "unexpected block key: {}. block key should only be 'config' or 'test'.".format(key)
) )
return testset return testcase
def _get_block_by_name(ref_call, ref_type): def _get_block_by_name(ref_call, ref_type):
""" get test content by reference name """ get test content by reference name.
@params:
ref_call: e.g. api_v1_Account_Login_POST($UserName, $Password) Args:
ref_type: "api" or "suite" ref_call (str): call function.
e.g. api_v1_Account_Login_POST($UserName, $Password)
ref_type (enum): "def-api" or "def-testcase"
Returns:
dict: api/testcase definition.
Raises:
exceptions.ParamsError: call args number is not equal to defined args number.
""" """
function_meta = parser.parse_function(ref_call) function_meta = parser.parse_function(ref_call)
func_name = function_meta["func_name"] func_name = function_meta["func_name"]
call_args = function_meta["args"] call_args = function_meta["args"]
block = _get_test_definition(func_name, ref_type) block = _get_test_definition(func_name, ref_type)
def_args = block.get("function_meta").get("args", []) def_args = block.get("function_meta", {}).get("args", [])
if len(call_args) != len(def_args): if len(call_args) != len(def_args):
raise exceptions.ParamsError("call args mismatch defined args!") err_msg = "{}: call args number is not equal to defined args number!\n".format(func_name)
err_msg += "defined args: {}\n".format(def_args)
err_msg += "reference args: {}".format(call_args)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
args_mapping = {} args_mapping = {}
for index, item in enumerate(def_args): for index, item in enumerate(def_args):
@@ -542,80 +508,104 @@ def _get_block_by_name(ref_call, ref_type):
def _get_test_definition(name, ref_type): def _get_test_definition(name, ref_type):
""" get expected api or testcase. """ get expected api or testcase.
@params:
name: api or testcase name Args:
ref_type: "api" or "suite" name (str): api or testcase name
@return ref_type (enum): "def-api" or "def-testcase"
expected api info if found, otherwise raise ApiNotFound exception
Returns:
dict: expected api/testcase info if found.
Raises:
exceptions.ApiNotFound: api not found
exceptions.TestcaseNotFound: testcase not found
""" """
block = overall_def_dict.get(ref_type, {}).get(name) block = project_mapping.get(ref_type, {}).get(name)
if not block: if not block:
err_msg = "{} not found!".format(name) err_msg = "{} not found!".format(name)
if ref_type == "api": if ref_type == "def-api":
raise exceptions.ApiNotFound(err_msg) raise exceptions.ApiNotFound(err_msg)
else: else:
# ref_type == "suite": # ref_type == "def-testcase":
raise exceptions.TestcaseNotFound(err_msg) raise exceptions.TestcaseNotFound(err_msg)
return block return block
def _override_block(def_block, current_block): def _extend_block(ref_block, def_block):
""" override def_block with current_block """ extend ref_block with def_block.
@param def_block:
{ Args:
"name": "get token", def_block (dict): api definition dict.
"request": {...}, ref_block (dict): reference block
"validate": [{'eq': ['status_code', 200]}]
} Returns:
@param current_block: dict: extended reference block.
{
"name": "get token", Examples:
"extract": [{"token": "content.token"}], >>> def_block = {
"validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] "name": "get token 1",
} "request": {...},
@return "validate": [{'eq': ['status_code', 200]}]
{ }
"name": "get token", >>> ref_block = {
"request": {...}, "name": "get token 2",
"extract": [{"token": "content.token"}], "extract": [{"token": "content.token"}],
"validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]
} }
>>> _extend_block(def_block, ref_block)
{
"name": "get token 2",
"request": {...},
"extract": [{"token": "content.token"}],
"validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]
}
""" """
# TODO: override variables
def_validators = def_block.get("validate") or def_block.get("validators", []) def_validators = def_block.get("validate") or def_block.get("validators", [])
current_validators = current_block.get("validate") or current_block.get("validators", []) ref_validators = ref_block.get("validate") or ref_block.get("validators", [])
def_extrators = def_block.get("extract") \ def_extrators = def_block.get("extract") \
or def_block.get("extractors") \ or def_block.get("extractors") \
or def_block.get("extract_binds", []) or def_block.get("extract_binds", [])
current_extractors = current_block.get("extract") \ ref_extractors = ref_block.get("extract") \
or current_block.get("extractors") \ or ref_block.get("extractors") \
or current_block.get("extract_binds", []) or ref_block.get("extract_binds", [])
current_block.update(def_block) ref_block.update(def_block)
current_block["validate"] = _merge_validator( ref_block["validate"] = _merge_validator(
def_validators, def_validators,
current_validators ref_validators
) )
current_block["extract"] = _merge_extractor( ref_block["extract"] = _merge_extractor(
def_extrators, def_extrators,
current_extractors ref_extractors
) )
def _get_validators_mapping(validators): def _convert_validators_to_mapping(validators):
""" get validators mapping from api or test validators """ convert validators list to mapping.
@param (list) validators:
[ Args:
{"check": "v1", "expect": 201, "comparator": "eq"}, validators (list): validators in list
{"check": {"b": 1}, "expect": 200, "comparator": "eq"}
] Returns:
@return dict: validators mapping, use (check, comparator) as key.
{
("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"}, Examples:
('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"} >>> validators = [
} {"check": "v1", "expect": 201, "comparator": "eq"},
{"check": {"b": 1}, "expect": 200, "comparator": "eq"}
]
>>> _convert_validators_to_mapping(validators)
{
("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
}
""" """
validators_mapping = {} validators_mapping = {}
@@ -633,48 +623,66 @@ def _get_validators_mapping(validators):
return validators_mapping return validators_mapping
def _merge_validator(def_validators, current_validators): def _merge_validator(def_validators, ref_validators):
""" merge def_validators with current_validators """ merge def_validators with ref_validators.
@params:
def_validators: [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] Args:
current_validators: [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] def_validators (list):
@return: ref_validators (list):
[
{"check": "v1", "expect": 201, "comparator": "eq"}, Returns:
{"check": "s2", "expect": 16, "comparator": "len_eq"}, list: merged validators
{"check": "s3", "expect": 12, "comparator": "len_eq"}
] Examples:
>>> def_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}]
>>> ref_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}]
>>> _merge_validator(def_validators, ref_validators)
[
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": "s2", "expect": 16, "comparator": "len_eq"},
{"check": "s3", "expect": 12, "comparator": "len_eq"}
]
""" """
if not def_validators: if not def_validators:
return current_validators return ref_validators
elif not current_validators: elif not ref_validators:
return def_validators return def_validators
else: else:
api_validators_mapping = _get_validators_mapping(def_validators) def_validators_mapping = _convert_validators_to_mapping(def_validators)
test_validators_mapping = _get_validators_mapping(current_validators) ref_validators_mapping = _convert_validators_to_mapping(ref_validators)
api_validators_mapping.update(test_validators_mapping) def_validators_mapping.update(ref_validators_mapping)
return list(api_validators_mapping.values()) return list(def_validators_mapping.values())
def _merge_extractor(def_extrators, current_extractors): def _merge_extractor(def_extrators, ref_extractors):
""" merge def_extrators with current_extractors """ merge def_extrators with ref_extractors
@params:
def_extrators: [{"var1": "val1"}, {"var2": "val2"}] Args:
current_extractors: [{"var1": "val111"}, {"var3": "val3"}] def_extrators (list): [{"var1": "val1"}, {"var2": "val2"}]
@return: ref_extractors (list): [{"var1": "val111"}, {"var3": "val3"}]
[
{"var1": "val111"}, Returns:
{"var2": "val2"}, list: merged extractors
{"var3": "val3"}
] Examples:
>>> def_extrators = [{"var1": "val1"}, {"var2": "val2"}]
>>> ref_extractors = [{"var1": "val111"}, {"var3": "val3"}]
>>> _merge_extractor(def_extrators, ref_extractors)
[
{"var1": "val111"},
{"var2": "val2"},
{"var3": "val3"}
]
""" """
if not def_extrators: if not def_extrators:
return current_extractors return ref_extractors
elif not current_extractors: elif not ref_extractors:
return def_extrators return def_extrators
else: else:
@@ -687,7 +695,7 @@ def _merge_extractor(def_extrators, current_extractors):
var_name = list(api_extrator.keys())[0] var_name = list(api_extrator.keys())[0]
extractor_dict[var_name] = api_extrator[var_name] extractor_dict[var_name] = api_extrator[var_name]
for test_extrator in current_extractors: for test_extrator in ref_extractors:
if len(test_extrator) != 1: if len(test_extrator) != 1:
logger.log_warning("incorrect extractor: {}".format(test_extrator)) logger.log_warning("incorrect extractor: {}".format(test_extrator))
continue continue
@@ -702,63 +710,6 @@ def _merge_extractor(def_extrators, current_extractors):
return extractor_list return extractor_list
def load_testcases(path):
""" load testcases from file path
Args:
path (str): testcase file/foler path.
path could be in several types:
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
Returns:
list: testcases list, each testcase is corresponding to a file
[
testcase_dict_1,
testcase_dict_2
]
"""
if isinstance(path, (list, set)):
testcases_list = []
for file_path in set(path):
testcases = load_testcases(file_path)
if not testcases:
continue
testcases_list.extend(testcases)
return testcases_list
if not os.path.isabs(path):
path = os.path.join(os.getcwd(), path)
if path in testcases_cache_mapping:
return testcases_cache_mapping[path]
if os.path.isdir(path):
files_list = load_folder_files(path)
testcases_list = load_testcases(files_list)
elif os.path.isfile(path):
try:
testcase = _load_test_file(path)
if testcase["testcases"]:
testcases_list = [testcase]
else:
testcases_list = []
except exceptions.FileFormatError:
testcases_list = []
else:
err_msg = "path not exist: {}".format(path)
logger.log_error(err_msg)
raise exceptions.FileNotFound(err_msg)
testcases_cache_mapping[path] = testcases_list
return testcases_list
def load_folder_content(folder_path): def load_folder_content(folder_path):
""" load api/testcases/testsuites definitions from folder. """ load api/testcases/testsuites definitions from folder.
@@ -843,7 +794,7 @@ def load_api_folder(api_folder_path=None):
api_dict["function_meta"] = function_meta api_dict["function_meta"] = function_meta
api_definition_mapping[func_name] = api_dict api_definition_mapping[func_name] = api_dict
project_mapping["tests"]["api"] = api_definition_mapping project_mapping["def-api"] = api_definition_mapping
return api_definition_mapping return api_definition_mapping
@@ -874,7 +825,7 @@ def load_test_folder(test_folder_path=None):
dict: testcases definition mapping. dict: testcases definition mapping.
{ {
"tests/testcases/setup.yml": [ "create_and_check": [
{"config": {}}, {"config": {}},
{"test": {}}, {"test": {}},
{"test": {}} {"test": {}}
@@ -900,7 +851,7 @@ def load_test_folder(test_folder_path=None):
"config": { "config": {
"path": test_file_path "path": test_file_path
}, },
"tests": [] "teststeps": []
} }
for item in items: for item in items:
key, block = item.popitem() key, block = item.popitem()
@@ -919,13 +870,13 @@ def load_test_folder(test_folder_path=None):
if func_name in test_definition_mapping: if func_name in test_definition_mapping:
logger.log_warning("API definition duplicated: {}".format(func_name)) logger.log_warning("API definition duplicated: {}".format(func_name))
block["function_meta"] = function_meta testcase["function_meta"] = function_meta
test_definition_mapping[func_name] = testcase test_definition_mapping[func_name] = testcase
else: else:
# key == "test": # key == "test":
testcase["tests"].append(block) testcase["teststeps"].append(block)
project_mapping["tests"]["testcases"] = test_definition_mapping project_mapping["def-testcase"] = test_definition_mapping
return test_definition_mapping return test_definition_mapping
@@ -949,15 +900,59 @@ def load_project_tests(folder_path=None):
return project_mapping return project_mapping
def load(path): def load_testcases(path):
""" main interface for loading testcases """ load testcases from file path
Args: Args:
path (str): testcase file/folder path path (str): testcase file/foler path.
path could be in several types:
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
Returns: Returns:
list: testcases list list: testcases list, each testcase is corresponding to a file
[
testcase_dict_1,
testcase_dict_2
]
""" """
_load_test_dependencies() if isinstance(path, (list, set)):
return load_testcases(path) testcases_list = []
for file_path in set(path):
testcases = load_testcases(file_path)
if not testcases:
continue
testcases_list.extend(testcases)
return testcases_list
if not os.path.isabs(path):
path = os.path.join(os.getcwd(), path)
if path in testcases_cache_mapping:
return testcases_cache_mapping[path]
if os.path.isdir(path):
files_list = load_folder_files(path)
testcases_list = load_testcases(files_list)
elif os.path.isfile(path):
try:
testcase = _load_test_file(path)
if testcase["teststeps"]:
testcases_list = [testcase]
else:
testcases_list = []
except exceptions.FileFormatError:
testcases_list = []
else:
err_msg = "path not exist: {}".format(path)
logger.log_error(err_msg)
raise exceptions.FileNotFound(err_msg)
testcases_cache_mapping[path] = testcases_list
return testcases_list

2
httprunner/locusts.py
View File

@@ -40,7 +40,7 @@ def gen_locustfile(testcase_file_path):
"templates", "templates",
"locustfile_template" "locustfile_template"
) )
testcases = loader.load(testcase_file_path) testcases = loader.load_testcases(testcase_file_path)
host = testcases[0].get("config", {}).get("request", {}).get("base_url", "") host = testcases[0].get("config", {}).get("request", {}).get("base_url", "")
with io.open(template_path, encoding='utf-8') as template: with io.open(template_path, encoding='utf-8') as template:

61
httprunner/task.py
View File

@@ -34,39 +34,39 @@ class TestCase(unittest.TestCase):
class TestSuite(unittest.TestSuite): class TestSuite(unittest.TestSuite):
""" create test suite with a testset, it may include one or several testcases. """ create test suite with a testcase, it may include one or several teststeps.
each suite should initialize a separate Runner() with testset config. each suite should initialize a separate Runner() with testcase config.
@param
(dict) testset Args:
testcase (dict): testcase dict
{ {
"name": "testset description",
"config": { "config": {
"name": "testset description", "name": "testcase description",
"parameters": {}, "parameters": {},
"variables": [], "variables": [],
"request": {}, "request": {},
"output": [] "output": []
}, },
"testcases": [ "teststeps": [
{ {
"name": "testcase description", "name": "teststep1 description",
"parameters": {}, "parameters": {},
"variables": [], # optional, override "variables": [], # optional, override
"request": {}, "request": {},
"extract": {}, # optional "extract": {}, # optional
"validate": {} # optional "validate": {} # optional
}, },
testcase12 teststep2
] ]
} }
(dict) variables_mapping: variables_mapping (dict): passed in variables mapping, it will override variables in config block.
passed in variables mapping, it will override variables in config block
""" """
def __init__(self, testset, variables_mapping=None, http_client_session=None): def __init__(self, testcase, variables_mapping=None, http_client_session=None):
super(TestSuite, self).__init__() super(TestSuite, self).__init__()
self.test_runner_list = [] self.test_runner_list = []
self.config = testset.get("config", {}) self.config = testcase.get("config", {})
self.output_variables_list = self.config.get("output", []) self.output_variables_list = self.config.get("output", [])
self.testset_file_path = self.config.get("path") self.testset_file_path = self.config.get("path")
config_dict_parameters = self.config.get("parameters", []) config_dict_parameters = self.config.get("parameters", [])
@@ -80,22 +80,22 @@ class TestSuite(unittest.TestSuite):
config_dict_parameters config_dict_parameters
) )
self.testcase_parser = context.TestcaseParser() self.testcase_parser = context.TestcaseParser()
testcases = testset.get("testcases", []) teststeps = testcase.get("teststeps", [])
for config_variables in config_parametered_variables_list: for config_variables in config_parametered_variables_list:
# config level # config level
self.config["variables"] = config_variables self.config["variables"] = config_variables
test_runner = runner.Runner(self.config, http_client_session) test_runner = runner.Runner(self.config, http_client_session)
for testcase_dict in testcases: for teststep_dict in teststeps:
testcase_dict = copy.copy(testcase_dict) teststep_dict = copy.copy(teststep_dict)
# testcase level # testcase level
testcase_parametered_variables_list = self._get_parametered_variables( testcase_parametered_variables_list = self._get_parametered_variables(
testcase_dict.get("variables", []), teststep_dict.get("variables", []),
testcase_dict.get("parameters", []) teststep_dict.get("parameters", [])
) )
for testcase_variables in testcase_parametered_variables_list: for testcase_variables in testcase_parametered_variables_list:
testcase_dict["variables"] = testcase_variables teststep_dict["variables"] = testcase_variables
# eval testcase name with bind variables # eval testcase name with bind variables
variables = utils.override_variables_binds( variables = utils.override_variables_binds(
@@ -104,13 +104,13 @@ class TestSuite(unittest.TestSuite):
) )
self.testcase_parser.update_binded_variables(variables) self.testcase_parser.update_binded_variables(variables)
try: try:
testcase_name = self.testcase_parser.eval_content_with_bindings(testcase_dict["name"]) testcase_name = self.testcase_parser.eval_content_with_bindings(teststep_dict["name"])
except (AssertionError, exceptions.ParamsError): except (AssertionError, exceptions.ParamsError):
logger.log_warning("failed to eval testcase name: {}".format(testcase_dict["name"])) logger.log_warning("failed to eval testcase name: {}".format(teststep_dict["name"]))
testcase_name = testcase_dict["name"] testcase_name = teststep_dict["name"]
self.test_runner_list.append((test_runner, variables)) self.test_runner_list.append((test_runner, variables))
self._add_test_to_suite(testcase_name, test_runner, testcase_dict) self._add_test_to_suite(testcase_name, test_runner, teststep_dict)
def _get_parametered_variables(self, variables, parameters): def _get_parametered_variables(self, variables, parameters):
""" parameterize varaibles with parameters """ parameterize varaibles with parameters
@@ -166,15 +166,14 @@ def init_test_suites(path_or_testcases, mapping=None, http_client_session=None):
Args: Args:
path_or_testcases (str/dict/list): testcase file path or testcase dict or testcases list path_or_testcases (str/dict/list): testcase file path or testcase dict or testcases list
testset_dict testcase_dict
or or
[ [
testset_dict_1, testcase_dict_1,
testset_dict_2, testcase_dict_2,
{ {
"config": {}, "config": {},
"api": {}, "teststeps": [teststep11, teststep12]
"testcases": [testcase11, testcase12]
} }
] ]
@@ -188,7 +187,7 @@ def init_test_suites(path_or_testcases, mapping=None, http_client_session=None):
if validator.is_testcases(path_or_testcases): if validator.is_testcases(path_or_testcases):
testcases = path_or_testcases testcases = path_or_testcases
else: else:
testcases = loader.load(path_or_testcases) testcases = loader.load_testcases(path_or_testcases)
# TODO: move comparator uniform here # TODO: move comparator uniform here
mapping = mapping or {} mapping = mapping or {}
@@ -237,12 +236,12 @@ class HttpRunner(object):
""" start to run test with varaibles mapping. """ start to run test with varaibles mapping.
Args: Args:
path_or_testcases (str/list/dict): YAML/JSON testset file path or testset list path_or_testcases (str/list/dict): YAML/JSON testcase file path or testcase list
path: path could be in several type path: path could be in several type
- absolute/relative file path - absolute/relative file path
- absolute/relative folder path - absolute/relative folder path
- list/set container with file(s) and/or folder(s) - list/set container with file(s) and/or folder(s)
testsets: testset or list of testset testcases: testcase dict or list of testcases
- (dict) testset_dict - (dict) testset_dict
- (list) list of testset_dict - (list) list of testset_dict
[ [

49
httprunner/validator.py
View File

@@ -6,35 +6,48 @@ TODO: refactor with JSON schema validate
""" """
def is_testcase(data_structure): def is_testcase(data_structure):
""" check if data_structure is a testcase """ check if data_structure is a testcase.
testcase should always be in the following data structure:
{ Args:
"name": "desc1", data_structure (dict): testcase should always be in the following data structure:
"config": {},
"api": {}, {
"testcases": [testcase11, testcase12] "name": "desc1",
} "config": {},
"api": {},
"testcases": [testcase11, testcase12]
}
Returns:
bool: True if data_structure is valid testcase, otherwise False.
""" """
if not isinstance(data_structure, dict): if not isinstance(data_structure, dict):
return False return False
if "name" not in data_structure or "testcases" not in data_structure: if "name" not in data_structure or "teststeps" not in data_structure:
return False return False
if not isinstance(data_structure["testcases"], list): if not isinstance(data_structure["teststeps"], list):
return False return False
return True return True
def is_testcases(data_structure): def is_testcases(data_structure):
""" check if data_structure is testcase or testcases list """ check if data_structure is testcase or testcases list.
testsets should always be in the following data structure:
testset_dict Args:
or data_structure (dict): testcase(s) should always be in the following data structure:
[
testset_dict_1, testcase_dict
testset_dict_2 or
] [
testcase_dict_1,
testcase_dict_2
]
Returns:
bool: True if data_structure is valid testcase(s), otherwise False.
""" """
if not isinstance(data_structure, list): if not isinstance(data_structure, list):
return is_testcase(data_structure) return is_testcase(data_structure)

4
tests/test_httprunner.py
View File

@@ -23,7 +23,7 @@ class TestHttpRunner(ApiServerUnittest):
'output': ['token'] 'output': ['token']
}, },
'api': {}, 'api': {},
'testcases': [ 'teststeps': [
{ {
'name': '/api/get-token', 'name': '/api/get-token',
'request': { 'request': {
@@ -114,7 +114,7 @@ class TestHttpRunner(ApiServerUnittest):
testsets = [ testsets = [
{ {
"name": "post data", "name": "post data",
"testcases": [ "teststeps": [
{ {
"name": "post data", "name": "post data",
"request": { "request": {

98
tests/test_loader.py
View File

@@ -219,60 +219,34 @@ class TestModuleLoader(unittest.TestCase):
class TestSuiteLoader(unittest.TestCase): class TestSuiteLoader(unittest.TestCase):
def setUp(self): @classmethod
loader.overall_def_dict = { def setUpClass(cls):
"api": {}, project_dir = os.path.join(os.getcwd(), "tests")
"suite": {} loader.load_project_tests(project_dir)
}
def test_load_test_dependencies(self):
loader._load_test_dependencies()
overall_def_dict = loader.overall_def_dict
self.assertIn("get_token", overall_def_dict["api"])
self.assertIn("create_and_check", overall_def_dict["suite"])
def test_load_api_file(self):
loader._load_api_file("tests/api/basic.yml")
overall_api_def_dict = loader.overall_def_dict["api"]
self.assertIn("get_token",overall_api_def_dict)
self.assertEqual("/api/get-token", overall_api_def_dict["get_token"]["request"]["url"])
self.assertIn("$user_agent", overall_api_def_dict["get_token"]["function_meta"]["args"])
self.assertEqual(len(overall_api_def_dict["get_token"]["validate"]), 3)
def test_load_test_file_suite(self):
loader._load_api_file("tests/api/basic.yml")
testset = loader._load_test_file("tests/suite/create_and_get.yml")
self.assertEqual(testset["config"]["name"], "create user and check result.")
self.assertEqual(len(testset["testcases"]), 3)
self.assertEqual(testset["testcases"][0]["name"], "make sure user $uid does not exist")
self.assertEqual(testset["testcases"][0]["request"]["url"], "/api/users/$uid")
def test_load_test_file_testcase(self): def test_load_test_file_testcase(self):
loader._load_test_dependencies() testcase = loader._load_test_file("tests/testcases/smoketest.yml")
testset = loader._load_test_file("tests/testcases/smoketest.yml") self.assertEqual(testcase["config"]["name"], "smoketest")
self.assertEqual(testset["config"]["name"], "smoketest") self.assertEqual(testcase["config"]["path"], "tests/testcases/smoketest.yml")
self.assertEqual(testset["config"]["path"], "tests/testcases/smoketest.yml") self.assertIn("device_sn", testcase["config"]["variables"][0])
self.assertIn("device_sn", testset["config"]["variables"][0]) self.assertEqual(len(testcase["teststeps"]), 8)
self.assertEqual(len(testset["testcases"]), 8) self.assertEqual(testcase["teststeps"][0]["name"], "get token")
self.assertEqual(testset["testcases"][0]["name"], "get token")
def test_get_block_by_name(self): def test_get_block_by_name(self):
loader._load_test_dependencies()
ref_call = "get_user($uid, $token)" ref_call = "get_user($uid, $token)"
block = loader._get_block_by_name(ref_call, "api") block = loader._get_block_by_name(ref_call, "def-api")
self.assertEqual(block["request"]["url"], "/api/users/$uid") self.assertEqual(block["request"]["url"], "/api/users/$uid")
self.assertEqual(block["function_meta"]["func_name"], "get_user") self.assertEqual(block["function_meta"]["func_name"], "get_user")
self.assertEqual(block["function_meta"]["args"], ['$uid', '$token']) self.assertEqual(block["function_meta"]["args"], ['$uid', '$token'])
def test_get_block_by_name_args_mismatch(self): def test_get_block_by_name_args_mismatch(self):
loader._load_test_dependencies()
ref_call = "get_user($uid, $token, $var)" ref_call = "get_user($uid, $token, $var)"
with self.assertRaises(exceptions.ParamsError): with self.assertRaises(exceptions.ParamsError):
loader._get_block_by_name(ref_call, "api") loader._get_block_by_name(ref_call, "def-api")
def test_override_block(self): def test_override_block(self):
loader._load_test_dependencies() def_block = loader._get_block_by_name(
def_block = loader._get_block_by_name("get_token($user_agent, $device_sn, $os_platform, $app_version)", "api") "get_token($user_agent, $device_sn, $os_platform, $app_version)", "def-api")
test_block = { test_block = {
"name": "override block", "name": "override block",
"variables": [ "variables": [
@@ -286,28 +260,26 @@ class TestSuiteLoader(unittest.TestCase):
] ]
} }
loader._override_block(def_block, test_block) loader._extend_block(test_block, def_block)
self.assertEqual(test_block["name"], "override block") self.assertEqual(test_block["name"], "override block")
self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, test_block["validate"]) self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, test_block["validate"])
self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, test_block["validate"]) self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, test_block["validate"])
def test_get_test_definition_api(self): def test_get_test_definition_api(self):
loader._load_test_dependencies() api_def = loader._get_test_definition("get_headers", "def-api")
api_def = loader._get_test_definition("get_headers", "api")
self.assertEqual(api_def["request"]["url"], "/headers") self.assertEqual(api_def["request"]["url"], "/headers")
self.assertEqual(len(api_def["setup_hooks"]), 2) self.assertEqual(len(api_def["setup_hooks"]), 2)
self.assertEqual(len(api_def["teardown_hooks"]), 1) self.assertEqual(len(api_def["teardown_hooks"]), 1)
with self.assertRaises(exceptions.ApiNotFound): with self.assertRaises(exceptions.ApiNotFound):
loader._get_test_definition("get_token_XXX", "api") loader._get_test_definition("get_token_XXX", "def-api")
def test_get_test_definition_suite(self): def test_get_test_definition_suite(self):
loader._load_test_dependencies() api_def = loader._get_test_definition("create_and_check", "def-testcase")
api_def = loader._get_test_definition("create_and_check", "suite")
self.assertEqual(api_def["config"]["name"], "create user and check result.") self.assertEqual(api_def["config"]["name"], "create user and check result.")
with self.assertRaises(exceptions.TestcaseNotFound): with self.assertRaises(exceptions.TestcaseNotFound):
loader._get_test_definition("create_and_check_XXX", "suite") loader._get_test_definition("create_and_check_XXX", "def-testcase")
def test_merge_validator(self): def test_merge_validator(self):
def_validators = [ def_validators = [
@@ -376,7 +348,7 @@ class TestSuiteLoader(unittest.TestCase):
self.assertEqual(len(testset_list), 1) self.assertEqual(len(testset_list), 1)
self.assertIn("path", testset_list[0]["config"]) self.assertIn("path", testset_list[0]["config"])
self.assertEqual(testset_list[0]["config"]["path"], path) self.assertEqual(testset_list[0]["config"]["path"], path)
self.assertEqual(len(testset_list[0]["testcases"]), 3) self.assertEqual(len(testset_list[0]["teststeps"]), 3)
testsets_list.extend(testset_list) testsets_list.extend(testset_list)
# relative file path # relative file path
@@ -385,7 +357,7 @@ class TestSuiteLoader(unittest.TestCase):
self.assertEqual(len(testset_list), 1) self.assertEqual(len(testset_list), 1)
self.assertIn("path", testset_list[0]["config"]) self.assertIn("path", testset_list[0]["config"])
self.assertIn(path, testset_list[0]["config"]["path"]) self.assertIn(path, testset_list[0]["config"]["path"])
self.assertEqual(len(testset_list[0]["testcases"]), 3) self.assertEqual(len(testset_list[0]["teststeps"]), 3)
testsets_list.extend(testset_list) testsets_list.extend(testset_list)
# list/set container with file(s) # list/set container with file(s)
@@ -395,20 +367,19 @@ class TestSuiteLoader(unittest.TestCase):
] ]
testset_list = loader.load_testcases(path) testset_list = loader.load_testcases(path)
self.assertEqual(len(testset_list), 2) self.assertEqual(len(testset_list), 2)
self.assertEqual(len(testset_list[0]["testcases"]), 3) self.assertEqual(len(testset_list[0]["teststeps"]), 3)
self.assertEqual(len(testset_list[1]["testcases"]), 3) self.assertEqual(len(testset_list[1]["teststeps"]), 3)
testsets_list.extend(testset_list) testsets_list.extend(testset_list)
self.assertEqual(len(testsets_list), 4) self.assertEqual(len(testsets_list), 4)
for testset in testsets_list: for testset in testsets_list:
for test in testset["testcases"]: for test in testset["teststeps"]:
self.assertIn('name', test) self.assertIn('name', test)
self.assertIn('request', test) self.assertIn('request', test)
self.assertIn('url', test['request']) self.assertIn('url', test['request'])
self.assertIn('method', test['request']) self.assertIn('method', test['request'])
def test_load_testcases_by_path_folder(self): def test_load_testcases_by_path_folder(self):
loader._load_test_dependencies()
# absolute folder path # absolute folder path
path = os.path.join(os.getcwd(), 'tests/data') path = os.path.join(os.getcwd(), 'tests/data')
testset_list_1 = loader.load_testcases(path) testset_list_1 = loader.load_testcases(path)
@@ -447,15 +418,14 @@ class TestSuiteLoader(unittest.TestCase):
loader.load_testcases(path) loader.load_testcases(path)
def test_load_testcases_by_path_layered(self): def test_load_testcases_by_path_layered(self):
loader._load_test_dependencies()
path = os.path.join( path = os.path.join(
os.getcwd(), 'tests/data/demo_testset_layer.yml') os.getcwd(), 'tests/data/demo_testset_layer.yml')
testsets_list = loader.load_testcases(path) testsets_list = loader.load_testcases(path)
self.assertIn("variables", testsets_list[0]["config"]) self.assertIn("variables", testsets_list[0]["config"])
self.assertIn("request", testsets_list[0]["config"]) self.assertIn("request", testsets_list[0]["config"])
self.assertIn("request", testsets_list[0]["testcases"][0]) self.assertIn("request", testsets_list[0]["teststeps"][0])
self.assertIn("url", testsets_list[0]["testcases"][0]["request"]) self.assertIn("url", testsets_list[0]["teststeps"][0]["request"])
self.assertIn("validate", testsets_list[0]["testcases"][0]) self.assertIn("validate", testsets_list[0]["teststeps"][0])
def test_load_folder_content(self): def test_load_folder_content(self):
path = os.path.join(os.getcwd(), "tests", "api") path = os.path.join(os.getcwd(), "tests", "api")
@@ -481,6 +451,10 @@ class TestSuiteLoader(unittest.TestCase):
testcases_definition_mapping["setup_and_reset"]["config"]["name"], testcases_definition_mapping["setup_and_reset"]["config"]["name"],
"setup and reset all." "setup and reset all."
) )
self.assertEqual(
testcases_definition_mapping["setup_and_reset"]["function_meta"]["func_name"],
"setup_and_reset"
)
def test_load_testsuites_folder(self): def test_load_testsuites_folder(self):
path = os.path.join(os.getcwd(), "tests", "testcases") path = os.path.join(os.getcwd(), "tests", "testcases")
@@ -500,18 +474,18 @@ class TestSuiteLoader(unittest.TestCase):
project_dir = os.path.join(os.getcwd(), "tests") project_dir = os.path.join(os.getcwd(), "tests")
project_tests = loader.load_project_tests(project_dir) project_tests = loader.load_project_tests(project_dir)
self.assertEqual(project_tests["debugtalk"]["variables"]["SECRET_KEY"], "DebugTalk") self.assertEqual(project_tests["debugtalk"]["variables"]["SECRET_KEY"], "DebugTalk")
self.assertIn("get_token", project_tests["tests"]["api"]) self.assertIn("get_token", project_tests["def-api"])
self.assertIn("setup_and_reset", project_tests["tests"]["testcases"]) self.assertIn("setup_and_reset", project_tests["def-testcase"])
def test_loader(self): def test_loader(self):
hrunner = task.HttpRunner(dot_env_path="tests/data/test.env") hrunner = task.HttpRunner(dot_env_path="tests/data/test.env")
self.assertEqual(hrunner.project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH") self.assertEqual(hrunner.project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH")
self.assertIn("debugtalk", hrunner.project_mapping) self.assertIn("debugtalk", hrunner.project_mapping)
self.assertIn("setup_and_reset", hrunner.project_mapping["tests"]["testcases"]) self.assertIn("setup_and_reset", hrunner.project_mapping["def-testcase"])
self.assertEqual( self.assertEqual(
hrunner.project_mapping["debugtalk"]["variables"]["SECRET_KEY"], hrunner.project_mapping["debugtalk"]["variables"]["SECRET_KEY"],
"DebugTalk" "DebugTalk"
) )
self.assertIn("get_sign", hrunner.project_mapping["debugtalk"]["functions"]) self.assertIn("get_sign", hrunner.project_mapping["debugtalk"]["functions"])
self.assertIn("get_token", hrunner.project_mapping["tests"]["api"]) self.assertIn("get_token", hrunner.project_mapping["def-api"])
self.assertIn("setup_and_reset", hrunner.project_mapping["tests"]["testcases"]) self.assertIn("setup_and_reset", hrunner.project_mapping["def-testcase"])

8
tests/test_runner.py
View File

@@ -170,7 +170,7 @@ class TestRunner(ApiServerUnittest):
"config": { "config": {
'path': 'tests/httpbin/hooks.yml', 'path': 'tests/httpbin/hooks.yml',
}, },
"testcases": [ "teststeps": [
{ {
"name": "test teardown hooks", "name": "test teardown hooks",
"request": { "request": {
@@ -206,7 +206,7 @@ class TestRunner(ApiServerUnittest):
"config": { "config": {
'path': 'tests/httpbin/hooks.yml', 'path': 'tests/httpbin/hooks.yml',
}, },
"testcases": [ "teststeps": [
{ {
"name": "test teardown hooks", "name": "test teardown hooks",
"request": { "request": {
@@ -236,7 +236,7 @@ class TestRunner(ApiServerUnittest):
"config": { "config": {
'path': 'tests/httpbin/hooks.yml', 'path': 'tests/httpbin/hooks.yml',
}, },
"testcases": [ "teststeps": [
{ {
"name": "test teardown hooks", "name": "test teardown hooks",
"request": { "request": {
@@ -384,7 +384,7 @@ class TestRunner(ApiServerUnittest):
testsets = loader.load_testcases(testcase_file_path) testsets = loader.load_testcases(testcase_file_path)
testset = testsets[0] testset = testsets[0]
config_dict_headers = testset["config"]["request"]["headers"] config_dict_headers = testset["config"]["request"]["headers"]
test_dict_headers = testset["testcases"][0]["request"]["headers"] test_dict_headers = testset["teststeps"][0]["request"]["headers"]
headers = deep_update_dict( headers = deep_update_dict(
config_dict_headers, config_dict_headers,
test_dict_headers test_dict_headers

2
tests/test_task.py
View File

@@ -37,7 +37,7 @@ class TestTask(ApiServerUnittest):
'output': ['token'] 'output': ['token']
}, },
'api': {}, 'api': {},
'testcases': [ 'teststeps': [
{ {
'name': '/api/get-token', 'name': '/api/get-token',
'request': { 'request': {