GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-09-04 23:16:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement lazy parser for validators

Browse Source
This commit is contained in:
debugtalk
2019-04-08 12:12:09 +08:00
parent 4b4ccf120f
commit 7a7c5503f8
15 changed files with 760 additions and 704 deletions
Download Patch File Download Diff File Expand all files Collapse all files
httprunner/api.py
+1 -3
View File
@@ -74,11 +74,9 @@ class HttpRunner(object):
return test return test
test_suite = unittest.TestSuite() test_suite = unittest.TestSuite()
functions = tests_mapping.get("project_mapping", {}).get("functions", {})
for testcase in tests_mapping["testcases"]: for testcase in tests_mapping["testcases"]:
config = testcase.get("config", {}) config = testcase.get("config", {})
test_runner = runner.Runner(config, functions) test_runner = runner.Runner(config)
TestSequense = type('TestSequense', (unittest.TestCase,), {}) TestSequense = type('TestSequense', (unittest.TestCase,), {})
tests = testcase.get("teststeps", []) tests = testcase.get("teststeps", [])
httprunner/cli.py
+2 -1
View File
@@ -8,8 +8,9 @@ def main_hrun():
from httprunner.__about__ import __description__, __version__ from httprunner.__about__ import __description__, __version__
from httprunner.api import HttpRunner from httprunner.api import HttpRunner
from httprunner.compat import is_py2 from httprunner.compat import is_py2
from httprunner.validator import validate_json_file
from httprunner.utils import (create_scaffold, get_python2_retire_msg, from httprunner.utils import (create_scaffold, get_python2_retire_msg,
prettify_json_file, validate_json_file) prettify_json_file)
parser = argparse.ArgumentParser(description=__description__) parser = argparse.ArgumentParser(description=__description__)
parser.add_argument( parser.add_argument(
httprunner/context.py
+67 -87
View File
@@ -5,18 +5,16 @@ class SessionContext(object):
""" HttpRunner session, store runtime variables. """ HttpRunner session, store runtime variables.
Examples: Examples:
>>> functions={...}
>>> variables = {"SECRET_KEY": "DebugTalk"} >>> variables = {"SECRET_KEY": "DebugTalk"}
>>> context = SessionContext(functions, variables) >>> context = SessionContext(variables)
Equivalent to: Equivalent to:
>>> context = SessionContext(functions) >>> context = SessionContext()
>>> context.update_session_variables(variables) >>> context.update_session_variables(variables)
""" """
def __init__(self, functions, variables=None): def __init__(self, variables=None):
self.session_variables_mapping = utils.ensure_mapping_format(variables or {}) self.session_variables_mapping = utils.ensure_mapping_format(variables or {})
self.FUNCTIONS_MAPPING = functions
self.init_test_variables() self.init_test_variables()
self.validation_results = [] self.validation_results = []
@@ -63,32 +61,20 @@ class SessionContext(object):
""" """
return parser.parse_lazy_data(content, self.test_variables_mapping) return parser.parse_lazy_data(content, self.test_variables_mapping)
def __eval_check_item(self, validator, resp_obj): def __eval_validator_check(self, check_item, resp_obj):
""" evaluate check item in validator. """ evaluate check item in validator.
Args: Args:
validator (dict): validator check_item: check_item should only be the following 5 formats:
{"check": "status_code", "comparator": "eq", "expect": 201} 1, variable reference, e.g. $token
{"check": "$resp_body_success", "comparator": "eq", "expect": True} 2, function reference, e.g. ${is_status_code_200($status_code)}
resp_obj (object): requests.Response() object 3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
4, string joined by delimiter. e.g. "status_code", "headers.content-type"
5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
Returns: resp_obj: response object
dict: validator info
{
"check": "status_code",
"check_value": 200,
"expect": 201,
"comparator": "eq"
}
""" """
check_item = validator["check"]
# check_item should only be the following 5 formats:
# 1, variable reference, e.g. $token
# 2, function reference, e.g. ${is_status_code_200($status_code)}
# 3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
# 4, string joined by delimiter. e.g. "status_code", "headers.content-type"
# 5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
if isinstance(check_item, (dict, list)) \ if isinstance(check_item, (dict, list)) \
or isinstance(check_item, parser.LazyString): or isinstance(check_item, parser.LazyString):
# format 1/2/3 # format 1/2/3
@@ -97,68 +83,22 @@ class SessionContext(object):
# format 4/5 # format 4/5
check_value = resp_obj.extract_field(check_item) check_value = resp_obj.extract_field(check_item)
validator["check_value"] = check_value return check_value
# expect_value should only be in 2 types: def __eval_validator_expect(self, expect_item):
# 1, variable reference, e.g. $expect_status_code """ evaluate expect item in validator.
# 2, actual value, e.g. 200
expect_value = self.eval_content(validator["expect"])
validator["expect"] = expect_value
validator["check_result"] = "unchecked"
return validator
def _do_validation(self, validator_dict):
""" validate with functions
Args: Args:
validator_dict (dict): validator dict expect_item: expect_item should only be in 2 types:
{ 1, variable reference, e.g. $expect_status_code
"check": "status_code", 2, actual value, e.g. 200
"check_value": 200,
"expect": 201,
"comparator": "eq"
}
""" """
# TODO: move comparator uniform to init_test_suites expect_value = self.eval_content(expect_item)
comparator = utils.get_uniform_comparator(validator_dict["comparator"]) return expect_value
validate_func = parser.get_mapping_function(comparator, self.FUNCTIONS_MAPPING)
check_item = validator_dict["check"]
check_value = validator_dict["check_value"]
expect_value = validator_dict["expect"]
if (check_value is None or expect_value is None) \
and comparator not in ["is", "eq", "equals", "=="]:
raise exceptions.ParamsError("Null value can only be compared with comparator: eq/equals/==")
validate_msg = "validate: {} {} {}({})".format(
check_item,
comparator,
expect_value,
type(expect_value).__name__
)
try:
validator_dict["check_result"] = "pass"
validate_func(check_value, expect_value)
validate_msg += "\t==> pass"
logger.log_debug(validate_msg)
except (AssertionError, TypeError):
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__,
comparator,
expect_value,
type(expect_value).__name__
)
logger.log_error(validate_msg)
validator_dict["check_result"] = "fail"
raise exceptions.ValidationFailure(validate_msg)
def validate(self, validators, resp_obj): def validate(self, validators, resp_obj):
""" make validations """ make validation with comparators
""" """
self.validation_results = [] self.validation_results = []
if not validators: if not validators:
@@ -170,19 +110,59 @@ class SessionContext(object):
failures = [] failures = []
for validator in validators: for validator in validators:
# evaluate validators with context variable mapping. # validator should be LazyFunction object
evaluated_validator = self.__eval_check_item( if not isinstance(validator, parser.LazyFunction):
parser.parse_validator(validator), raise exceptions.ValidationFailure(
"validator should be parsed first: {}".format(validators))
# evaluate validator args with context variable mapping.
validator_args = validator._args
check_item, expect_item = validator_args
check_value = self.__eval_validator_check(
check_item,
resp_obj resp_obj
) )
expect_value = self.__eval_validator_expect(expect_item)
validator._args = [check_value, expect_value]
comparator = validator._func.__name__
validator_dict = {
"comparator": comparator,
"check": check_item,
"check_value": check_value,
"expect": expect_item,
"expect_value": expect_value
}
validate_msg = "\nvalidate: {} {} {}({})".format(
check_item,
comparator,
expect_value,
type(expect_value).__name__
)
try: try:
self._do_validation(evaluated_validator) validator.to_value(self.test_variables_mapping)
except exceptions.ValidationFailure as ex: validator_dict["check_result"] = "pass"
validate_msg += "\t==> pass"
logger.log_debug(validate_msg)
except (AssertionError, TypeError):
validate_pass = False validate_pass = False
failures.append(str(ex)) validator_dict["check_result"] = "fail"
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__,
comparator,
expect_value,
type(expect_value).__name__
)
logger.log_error(validate_msg)
failures.append(validate_msg)
self.validation_results.append(evaluated_validator) self.validation_results.append(validator_dict)
# restore validator args, in case of running multiple times
validator._args = validator_args
if not validate_pass: if not validate_pass:
failures_string = "\n".join([failure for failure in failures]) failures_string = "\n".join([failure for failure in failures])
httprunner/parser.py
+39 -78
View File
@@ -4,7 +4,7 @@ import ast
import os import os
import re import re
from httprunner import exceptions, utils from httprunner import exceptions, utils, validator
from httprunner.compat import basestring, builtin_str, numeric_types, str from httprunner.compat import basestring, builtin_str, numeric_types, str
# TODO: change variable notation from $var to {{var}} # TODO: change variable notation from $var to {{var}}
@@ -105,65 +105,6 @@ def regex_findall_functions(content):
return [] return []
def parse_validator(validator):
""" parse validator
Args:
validator (dict): validator maybe in two formats:
format1: this is kept for compatiblity with the previous versions.
{"check": "status_code", "comparator": "eq", "expect": 201}
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
format2: recommended new version
{'eq': ['status_code', 201]}
{'eq': ['$resp_body_success', True]}
Returns
dict: validator info
{
"check": "status_code",
"expect": 201,
"comparator": "eq"
}
"""
if not isinstance(validator, dict):
raise exceptions.ParamsError("invalid validator: {}".format(validator))
if "check" in validator and len(validator) > 1:
# format1
check_item = validator.get("check")
if "expect" in validator:
expect_value = validator.get("expect")
elif "expected" in validator:
expect_value = validator.get("expected")
else:
raise exceptions.ParamsError("invalid validator: {}".format(validator))
comparator = validator.get("comparator", "eq")
elif len(validator) == 1:
# format2
comparator = list(validator.keys())[0]
compare_values = validator[comparator]
if not isinstance(compare_values, list) or len(compare_values) != 2:
raise exceptions.ParamsError("invalid validator: {}".format(validator))
check_item, expect_value = compare_values
else:
raise exceptions.ParamsError("invalid validator: {}".format(validator))
return {
"check": check_item,
"expect": expect_value,
"comparator": comparator
}
def parse_parameters(parameters, variables_mapping=None, functions_mapping=None): def parse_parameters(parameters, variables_mapping=None, functions_mapping=None):
""" parse parameters and generate cartesian product. """ parse parameters and generate cartesian product.
@@ -738,12 +679,9 @@ def _extend_with_api(test_dict, api_def_dict):
""" extend test with api definition, test will merge and override api definition. """ extend test with api definition, test will merge and override api definition.
Args: Args:
test_dict (dict): test block test_dict (dict): test block, this will override api_def_dict
api_def_dict (dict): api definition api_def_dict (dict): api definition
Returns:
dict: extended test dict.
Examples: Examples:
>>> api_def_dict = { >>> api_def_dict = {
"name": "get token 1", "name": "get token 1",
@@ -756,6 +694,7 @@ def _extend_with_api(test_dict, api_def_dict):
"validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]
} }
>>> _extend_with_api(test_dict, api_def_dict) >>> _extend_with_api(test_dict, api_def_dict)
>>> print(test_dict)
{ {
"name": "get token 2", "name": "get token 2",
"request": {...}, "request": {...},
@@ -764,9 +703,8 @@ def _extend_with_api(test_dict, api_def_dict):
} }
""" """
# override name # override api name
api_def_name = api_def_dict.pop("name", "") test_dict.setdefault("name", api_def_dict.pop("name", ""))
test_dict["name"] = test_dict.get("name") or api_def_name
# override variables # override variables
def_variables = api_def_dict.pop("variables", []) def_variables = api_def_dict.pop("variables", [])
@@ -777,16 +715,12 @@ def _extend_with_api(test_dict, api_def_dict):
# merge & override validators TODO: relocate # merge & override validators TODO: relocate
def_raw_validators = api_def_dict.pop("validate", []) def_raw_validators = api_def_dict.pop("validate", [])
ref_raw_validators = test_dict.get("validate", [])
def_validators = [ def_validators = [
parse_validator(validator) validator.uniform_validator(_validator)
for validator in def_raw_validators for _validator in def_raw_validators
] ]
ref_validators = [ ref_validators = test_dict.pop("validate", [])
parse_validator(validator) test_dict["validate"] = validator.extend_validators(
for validator in ref_raw_validators
]
test_dict["validate"] = utils.extend_validators(
def_validators, def_validators,
ref_validators ref_validators
) )
@@ -798,7 +732,7 @@ def _extend_with_api(test_dict, api_def_dict):
test_dict.get("extract", {}) test_dict.get("extract", {})
) )
# TODO: merge & override request # merge & override request
test_dict["request"] = api_def_dict.pop("request", {}) test_dict["request"] = api_def_dict.pop("request", {})
# base_url & verify: priority api_def_dict > test_dict # base_url & verify: priority api_def_dict > test_dict
@@ -824,8 +758,6 @@ def _extend_with_api(test_dict, api_def_dict):
# TODO: extend with other api definition items, e.g. times # TODO: extend with other api definition items, e.g. times
test_dict.update(api_def_dict) test_dict.update(api_def_dict)
return test_dict
def _extend_with_testcase(test_dict, testcase_def_dict): def _extend_with_testcase(test_dict, testcase_def_dict):
""" extend test with testcase definition """ extend test with testcase definition
@@ -923,6 +855,14 @@ def __prepare_testcase_tests(tests, config, project_mapping):
if (not test_dict.get("base_url")) and config_base_url: if (not test_dict.get("base_url")) and config_base_url:
test_dict["base_url"] = config_base_url test_dict["base_url"] = config_base_url
# unify validators' format
if "validate" in test_dict:
ref_raw_validators = test_dict.pop("validate", [])
test_dict["validate"] = [
validator.uniform_validator(_validator)
for _validator in ref_raw_validators
]
if "testcase_def" in test_dict: if "testcase_def" in test_dict:
# test_dict is nested testcase # test_dict is nested testcase
@@ -954,6 +894,27 @@ def __prepare_testcase_tests(tests, config, project_mapping):
check_variables_set = set(test_dict_variables.keys()) \ check_variables_set = set(test_dict_variables.keys()) \
| set(session_variables.keys()) | {"request", "response"} | set(session_variables.keys()) | {"request", "response"}
# convert validators to lazy function
validators = test_dict.pop("validate", [])
prepared_validators = []
for _validator in validators:
function_meta = {
"func_name": _validator["comparator"],
"args": [
_validator["check"],
_validator["expect"]
],
"kwargs": {}
}
prepared_validators.append(
LazyFunction(
function_meta,
functions,
check_variables_set
)
)
test_dict["validate"] = prepared_validators
# convert variables and functions to lazy object. # convert variables and functions to lazy object.
# raises VariableNotFound if undefined variable exists in test_dict # raises VariableNotFound if undefined variable exists in test_dict
prepared_test_dict = prepare_lazy_data( prepared_test_dict = prepare_lazy_data(
httprunner/runner.py
+31 -19
View File
@@ -11,27 +11,40 @@ class Runner(object):
""" Running testcases. """ Running testcases.
Examples: Examples:
>>> functions={...} >>> tests_mapping = {
>>> config = { "project_mapping": {
"name": "XXXX", "functions": {}
"base_url": "http://127.0.0.1", },
"verify": False "testcases": [
{
"config": {
"name": "XXXX",
"base_url": "http://127.0.0.1",
"verify": False
},
"teststeps": [
{
"name": "test description",
"variables": [], # optional
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "GET"
}
}
]
}
]
} }
>>> runner = Runner(config, functions)
>>> test_dict = { >>> parsed_tests_mapping = parser.parse_tests(tests_mapping)
"name": "test description", >>> parsed_testcase = parsed_tests_mapping["testcases"][0]
"variables": [], # optional
"request": { >>> test_runner = runner.Runner(parsed_testcase["config"])
"url": "http://127.0.0.1:5000/api/users/1000", >>> test_runner.run_test(parsed_testcase["teststeps"][0])
"method": "GET"
}
}
>>> runner.run_test(test_dict)
""" """
def __init__(self, config, functions, http_client_session=None): def __init__(self, config, http_client_session=None):
""" run testcase or testsuite. """ run testcase or testsuite.
Args: Args:
@@ -50,7 +63,6 @@ class Runner(object):
base_url = config.get("base_url") base_url = config.get("base_url")
self.verify = config.get("verify", True) self.verify = config.get("verify", True)
self.output = config.get("output", []) self.output = config.get("output", [])
self.functions = functions
self.validation_results = [] self.validation_results = []
# testcase setup hooks # testcase setup hooks
@@ -59,7 +71,7 @@ class Runner(object):
self.testcase_teardown_hooks = config.get("teardown_hooks", []) self.testcase_teardown_hooks = config.get("teardown_hooks", [])
self.http_client_session = http_client_session or HttpSession(base_url) self.http_client_session = http_client_session or HttpSession(base_url)
self.session_context = SessionContext(self.functions) self.session_context = SessionContext()
if testcase_setup_hooks: if testcase_setup_hooks:
self.do_hook_actions(testcase_setup_hooks, "setup") self.do_hook_actions(testcase_setup_hooks, "setup")
@@ -289,7 +301,7 @@ class Runner(object):
config = testcase_dict.get("config", {}) config = testcase_dict.get("config", {})
# each teststeps in one testcase (YAML/JSON) share the same session. # each teststeps in one testcase (YAML/JSON) share the same session.
test_runner = Runner(config, self.functions, self.http_client_session) test_runner = Runner(config, self.http_client_session)
tests = testcase_dict.get("teststeps", []) tests = testcase_dict.get("teststeps", [])
httprunner/templates/locustfile_template
+1 -2
View File
@@ -15,7 +15,7 @@ logging.getLogger('locust.runners').setLevel(logging.INFO)
class WebPageTasks(TaskSet): class WebPageTasks(TaskSet):
def on_start(self): def on_start(self):
self.test_runner = Runner(self.locust.config, self.locust.functions, self.client) self.test_runner = Runner(self.locust.config, self.client)
@task @task
def test_any(self): def test_any(self):
@@ -38,7 +38,6 @@ class WebPageUser(HttpLocust):
file_path = "$TESTCASE_FILE" file_path = "$TESTCASE_FILE"
locust_tests = prepare_locust_tests(file_path) locust_tests = prepare_locust_tests(file_path)
functions = locust_tests["functions"]
tests = locust_tests["tests"] tests = locust_tests["tests"]
config = {} config = {}
httprunner/utils.py
-124
View File
@@ -120,39 +120,6 @@ def query_json(json_content, query, delimiter='.'):
return json_content return json_content
def get_uniform_comparator(comparator):
""" convert comparator alias to uniform name
"""
if comparator in ["eq", "equals", "==", "is"]:
return "equals"
elif comparator in ["lt", "less_than"]:
return "less_than"
elif comparator in ["le", "less_than_or_equals"]:
return "less_than_or_equals"
elif comparator in ["gt", "greater_than"]:
return "greater_than"
elif comparator in ["ge", "greater_than_or_equals"]:
return "greater_than_or_equals"
elif comparator in ["ne", "not_equals"]:
return "not_equals"
elif comparator in ["str_eq", "string_equals"]:
return "string_equals"
elif comparator in ["len_eq", "length_equals", "count_eq"]:
return "length_equals"
elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]:
return "length_greater_than"
elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \
"count_greater_than_or_equals"]:
return "length_greater_than_or_equals"
elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]:
return "length_less_than"
elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \
"count_less_than_or_equals"]:
return "length_less_than_or_equals"
else:
return comparator
def deep_update_dict(origin_dict, override_dict): def deep_update_dict(origin_dict, override_dict):
""" update origin dict with override dict recursively """ update origin dict with override dict recursively
e.g. origin_dict = {'a': 1, 'b': {'c': 2, 'd': 4}} e.g. origin_dict = {'a': 1, 'b': {'c': 2, 'd': 4}}
@@ -323,78 +290,6 @@ def ensure_mapping_format(variables):
raise exceptions.ParamsError("variables format error!") raise exceptions.ParamsError("variables format error!")
def _convert_validators_to_mapping(validators):
""" convert validators list to mapping.
Args:
validators (list): validators in list
Returns:
dict: validators mapping, use (check, comparator) as key.
Examples:
>>> validators = [
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": {"b": 1}, "expect": 200, "comparator": "eq"}
]
>>> _convert_validators_to_mapping(validators)
{
("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
}
"""
validators_mapping = {}
for validator in validators:
if not isinstance(validator["check"], collections.Hashable):
check = json.dumps(validator["check"])
else:
check = validator["check"]
key = (check, validator["comparator"])
validators_mapping[key] = validator
return validators_mapping
def extend_validators(raw_validators, override_validators):
""" extend raw_validators with override_validators.
override_validators will merge and override raw_validators.
Args:
raw_validators (dict):
override_validators (dict):
Returns:
list: extended validators
Examples:
>>> raw_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}]
>>> override_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}]
>>> extend_validators(raw_validators, override_validators)
[
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": "s2", "expect": 16, "comparator": "len_eq"},
{"check": "s3", "expect": 12, "comparator": "len_eq"}
]
"""
if not raw_validators:
return override_validators
elif not override_validators:
return raw_validators
else:
def_validators_mapping = _convert_validators_to_mapping(raw_validators)
ref_validators_mapping = _convert_validators_to_mapping(override_validators)
def_validators_mapping.update(ref_validators_mapping)
return list(def_validators_mapping.values())
def extend_variables(raw_variables, override_variables): def extend_variables(raw_variables, override_variables):
""" extend raw_variables with override_variables. """ extend raw_variables with override_variables.
override_variables will merge and override raw_variables. override_variables will merge and override raw_variables.
@@ -581,25 +476,6 @@ def gen_cartesian_product(*args):
return product_list return product_list
def validate_json_file(file_list):
""" validate JSON testcase format
"""
for json_file in set(file_list):
if not json_file.endswith(".json"):
logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file))
continue
logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN")
with io.open(json_file) as stream:
try:
json.load(stream)
except ValueError as e:
raise SystemExit(e)
print("OK")
def prettify_json_file(file_list): def prettify_json_file(file_list):
""" prettify JSON testcase format """ prettify JSON testcase format
""" """
httprunner/validator.py
+188
View File
@@ -1,7 +1,12 @@
# encoding: utf-8 # encoding: utf-8
import collections
import io
import json
import os import os
import types import types
from httprunner import exceptions, logger
""" validate data format """ validate data format
TODO: refactor with JSON schema validate TODO: refactor with JSON schema validate
@@ -129,6 +134,170 @@ def is_testcase_path(path):
return True return True
###############################################################################
## testcase validator utils
###############################################################################
def get_uniform_comparator(comparator):
""" convert comparator alias to uniform name
"""
if comparator in ["eq", "equals", "==", "is"]:
return "equals"
elif comparator in ["lt", "less_than"]:
return "less_than"
elif comparator in ["le", "less_than_or_equals"]:
return "less_than_or_equals"
elif comparator in ["gt", "greater_than"]:
return "greater_than"
elif comparator in ["ge", "greater_than_or_equals"]:
return "greater_than_or_equals"
elif comparator in ["ne", "not_equals"]:
return "not_equals"
elif comparator in ["str_eq", "string_equals"]:
return "string_equals"
elif comparator in ["len_eq", "length_equals", "count_eq"]:
return "length_equals"
elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]:
return "length_greater_than"
elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \
"count_greater_than_or_equals"]:
return "length_greater_than_or_equals"
elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]:
return "length_less_than"
elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \
"count_less_than_or_equals"]:
return "length_less_than_or_equals"
else:
return comparator
def uniform_validator(validator):
""" unify validator
Args:
validator (dict): validator maybe in two formats:
format1: this is kept for compatiblity with the previous versions.
{"check": "status_code", "comparator": "eq", "expect": 201}
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
format2: recommended new version, {comparator: [check_item, expected_value]}
{'eq': ['status_code', 201]}
{'eq': ['$resp_body_success', True]}
Returns
dict: validator info
{
"check": "status_code",
"expect": 201,
"comparator": "equals"
}
"""
if not isinstance(validator, dict):
raise exceptions.ParamsError("invalid validator: {}".format(validator))
if "check" in validator and "expect" in validator:
# format1
check_item = validator["check"]
expect_value = validator["expect"]
comparator = validator.get("comparator", "eq")
elif len(validator) == 1:
# format2
comparator = list(validator.keys())[0]
compare_values = validator[comparator]
if not isinstance(compare_values, list) or len(compare_values) != 2:
raise exceptions.ParamsError("invalid validator: {}".format(validator))
check_item, expect_value = compare_values
else:
raise exceptions.ParamsError("invalid validator: {}".format(validator))
# uniform comparator, e.g. lt => less_than, eq => equals
comparator = get_uniform_comparator(comparator)
return {
"check": check_item,
"expect": expect_value,
"comparator": comparator
}
def _convert_validators_to_mapping(validators):
""" convert validators list to mapping.
Args:
validators (list): validators in list
Returns:
dict: validators mapping, use (check, comparator) as key.
Examples:
>>> validators = [
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": {"b": 1}, "expect": 200, "comparator": "eq"}
]
>>> _convert_validators_to_mapping(validators)
{
("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
}
"""
validators_mapping = {}
for validator in validators:
if not isinstance(validator["check"], collections.Hashable):
check = json.dumps(validator["check"])
else:
check = validator["check"]
key = (check, validator["comparator"])
validators_mapping[key] = validator
return validators_mapping
def extend_validators(raw_validators, override_validators):
""" extend raw_validators with override_validators.
override_validators will merge and override raw_validators.
Args:
raw_validators (dict):
override_validators (dict):
Returns:
list: extended validators
Examples:
>>> raw_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}]
>>> override_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}]
>>> extend_validators(raw_validators, override_validators)
[
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": "s2", "expect": 16, "comparator": "len_eq"},
{"check": "s3", "expect": 12, "comparator": "len_eq"}
]
"""
if not raw_validators:
return override_validators
elif not override_validators:
return raw_validators
else:
def_validators_mapping = _convert_validators_to_mapping(raw_validators)
ref_validators_mapping = _convert_validators_to_mapping(override_validators)
def_validators_mapping.update(ref_validators_mapping)
return list(def_validators_mapping.values())
############################################################################### ###############################################################################
## validate varibles and functions ## validate varibles and functions
############################################################################### ###############################################################################
@@ -157,3 +326,22 @@ def is_variable(tup):
return False return False
return True return True
def validate_json_file(file_list):
""" validate JSON testcase format
"""
for json_file in set(file_list):
if not json_file.endswith(".json"):
logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file))
continue
logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN")
with io.open(json_file) as stream:
try:
json.load(stream)
except ValueError as e:
raise SystemExit(e)
print("OK")
tests/httpbin/api/302_redirect.yml
+1
View File
@@ -6,5 +6,6 @@ request:
url: https://debugtalk.com url: https://debugtalk.com
status_code: 302 status_code: 302
method: GET method: GET
verify: False
validate: validate:
- eq: ["status_code", 200] - eq: ["status_code", 200]
tests/test_client.py
+1 -1
View File
@@ -76,7 +76,7 @@ class TestHttpClient(ApiServerUnittest):
"a": "1", "a": "1",
"b": "2" "b": "2"
} }
resp = self.api_client.get(url, cookies=cookies, headers=self.headers) resp = self.api_client.get(url, cookies=cookies, headers=self.headers, verify=False)
raw_request = resp.history[0].request raw_request = resp.history[0].request
self.assertEqual(raw_request._cookies["a"], "1") self.assertEqual(raw_request._cookies["a"], "1")
self.assertEqual(raw_request._cookies["b"], "2") self.assertEqual(raw_request._cookies["b"], "2")
tests/test_context.py
+70 -86
View File
@@ -1,8 +1,7 @@
import os import os
import time import time
import requests from httprunner import context, exceptions, loader, parser, runner
from httprunner import context, exceptions, loader, parser, response, utils
from tests.base import ApiServerUnittest, gen_md5, gen_random_string from tests.base import ApiServerUnittest, gen_md5, gen_random_string
@@ -11,16 +10,10 @@ class TestContext(ApiServerUnittest):
def setUp(self): def setUp(self):
loader.load_project_tests(os.path.join(os.getcwd(), "tests")) loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
project_mapping = loader.project_mapping project_mapping = loader.project_mapping
self.functions = project_mapping["functions"]
self.context = context.SessionContext( self.context = context.SessionContext(
functions=self.functions,
variables={"SECRET_KEY": "DebugTalk"} variables={"SECRET_KEY": "DebugTalk"}
) )
def test_init_context_functions(self):
context_functions = self.context.FUNCTIONS_MAPPING
self.assertIn("gen_md5", context_functions)
def test_init_test_variables_initialize(self): def test_init_test_variables_initialize(self):
self.assertEqual( self.assertEqual(
self.context.test_variables_mapping, self.context.test_variables_mapping,
@@ -57,13 +50,6 @@ class TestContext(ApiServerUnittest):
"debugtalk" "debugtalk"
) )
def test_eval_content_functions(self):
content = parser.prepare_lazy_data("${sleep_N_secs(1)}", self.functions)
start_time = time.time()
self.context.eval_content(content)
elapsed_time = time.time() - start_time
self.assertGreater(elapsed_time, 1)
def test_eval_content_variables(self): def test_eval_content_variables(self):
variables = { variables = {
"SECRET_KEY": "DebugTalk" "SECRET_KEY": "DebugTalk"
@@ -124,82 +110,80 @@ class TestContext(ApiServerUnittest):
) )
self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk") self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk")
def test_do_validation(self):
self.context._do_validation(
{"check": "check", "check_value": 1, "expect": 1, "comparator": "eq"}
)
self.context._do_validation(
{"check": "check", "check_value": "abc", "expect": "abc", "comparator": "=="}
)
self.context._do_validation(
{"check": "status_code", "check_value": "201", "expect": 3, "comparator": "sum_status_code"}
)
def test_validate(self): def test_validate(self):
url = "http://127.0.0.1:5000/" testcases = [
resp = requests.get(url) {
resp_obj = response.ResponseObject(resp) "config": {
'name': "test validation"
validators = [ },
{"eq": ["$resp_status_code", 201]}, "teststeps": [
{"check": "$resp_status_code", "comparator": "eq", "expect": 201}, {
{"check": "$resp_body_success", "comparator": "eq", "expect": True} "name": "test validation",
] "request": {
validators = parser.prepare_lazy_data(validators, {}, {"resp_status_code", "resp_body_success"}) "url": "http://127.0.0.1:5000/",
variables = { "method": "GET",
"resp_status_code": 200, },
"resp_body_success": True "variables": {
} "resp_status_code": 200,
"resp_body_success": True
self.context.init_test_variables(variables) },
"validate": [
with self.assertRaises(exceptions.ValidationFailure): {"eq": ["$resp_status_code", 200]},
self.context.validate(validators, resp_obj) {"check": "$resp_status_code", "comparator": "eq", "expect": 200},
{"check": "$resp_body_success", "expect": True},
validators = [ {"check": "${is_status_code_200($resp_status_code)}", "expect": True}
{"eq": ["$resp_status_code", 201]}, ]
{"check": "$resp_status_code", "comparator": "eq", "expect": 201}, }
{"check": "$resp_body_success", "comparator": "eq", "expect": True}, ]
{"check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False} }
] ]
from tests.debugtalk import is_status_code_200 from tests.debugtalk import is_status_code_200
functions = { tests_mapping = {
"is_status_code_200": is_status_code_200 "project_mapping": {
"functions": {
"is_status_code_200": is_status_code_200
}
},
"testcases": testcases
} }
validators = parser.prepare_lazy_data( parsed_tests_mapping = parser.parse_tests(tests_mapping)
validators, functions, {"resp_status_code", "resp_body_success"}) parsed_testcase = parsed_tests_mapping["testcases"][0]
variables = [ test_runner = runner.Runner(parsed_testcase["config"])
{"resp_status_code": 201}, teststep = parsed_testcase["teststeps"][0]
{"resp_body_success": True} test_runner.run_test(teststep)
]
self.context.init_test_variables(variables)
self.context.validate(validators, resp_obj)
self.context.validate([], resp_obj)
self.assertEqual(self.context.validation_results, [])
def test_validate_exception(self): def test_validate_exception(self):
url = "http://127.0.0.1:5000/" testcases = [
resp = requests.get(url) {
resp_obj = response.ResponseObject(resp) "config": {
'name': "test validation"
# expected value missed in validators },
validators = [ "teststeps": [
{"eq": ["$resp_status_code", 201]}, {
{"check": "$resp_status_code", "comparator": "eq", "expect": 201} "name": "test validation",
"request": {
"url": "http://127.0.0.1:5000/",
"method": "GET",
},
"variables": {
"resp_status_code": 200,
"resp_body_success": True
},
"validate": [
{"eq": ["$resp_status_code", 201]},
{"check": "$resp_status_code", "expect": 201},
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
]
}
]
}
] ]
validators = parser.prepare_lazy_data(validators, {}, {"resp_status_code"}) tests_mapping = {
variables = [] "testcases": testcases
self.context.init_test_variables(variables) }
parsed_tests_mapping = parser.parse_tests(tests_mapping)
with self.assertRaises(exceptions.VariableNotFound): parsed_testcase = parsed_tests_mapping["testcases"][0]
self.context.validate(validators, resp_obj) test_runner = runner.Runner(parsed_testcase["config"])
teststep = parsed_testcase["teststeps"][0]
# expected value missed in variables mapping
variables = [
{"resp_status_code": 200}
]
self.context.init_test_variables(variables)
with self.assertRaises(exceptions.ValidationFailure): with self.assertRaises(exceptions.ValidationFailure):
self.context.validate(validators, resp_obj) test_runner.run_test(teststep)
tests/test_parser.py
+9 -22
View File
@@ -104,19 +104,6 @@ class TestParserBasic(unittest.TestCase):
{'args': ["$request", '12 3'], 'kwargs': {}} {'args': ["$request", '12 3'], 'kwargs': {}}
) )
def test_parse_validator(self):
validator = {"check": "status_code", "comparator": "eq", "expect": 201}
self.assertEqual(
parser.parse_validator(validator),
{"check": "status_code", "comparator": "eq", "expect": 201}
)
validator = {'eq': ['status_code', 201]}
self.assertEqual(
parser.parse_validator(validator),
{"check": "status_code", "comparator": "eq", "expect": 201}
)
def test_extract_variables(self): def test_extract_variables(self):
prepared_content = parser.prepare_lazy_data("123$a", {}, {"a"}) prepared_content = parser.prepare_lazy_data("123$a", {}, {"a"})
self.assertEqual( self.assertEqual(
@@ -1076,15 +1063,15 @@ class TestParser(unittest.TestCase):
'json': {'sign': '${get_sign($device_sn, $os_platform, $app_version)}'} 'json': {'sign': '${get_sign($device_sn, $os_platform, $app_version)}'}
}, },
'validate': [ 'validate': [
{'eq': ['status_code', 201]}, {"check": "status_code", "comparator": "equals", "expect": 201},
{'len_eq': ['content.token', 32]} {"check": "content.token", "comparator": "length_equals", "expect": 32}
] ]
} }
extended_block = parser._extend_with_api(test_block, api_def_dict) parser._extend_with_api(test_block, api_def_dict)
self.assertEqual(extended_block["base_url"], "https://debugtalk.com") self.assertEqual(test_block["base_url"], "https://debugtalk.com")
self.assertEqual(extended_block["name"], "override block") self.assertEqual(test_block["name"], "override block")
self.assertEqual({'var': 123}, extended_block["variables"]) self.assertEqual({'var': 123}, test_block["variables"])
self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, extended_block["validate"]) self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'equals'}, test_block["validate"])
self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, extended_block["validate"]) self.assertIn({'check': 'content.token', 'comparator': 'length_equals', 'expect': 32}, test_block["validate"])
self.assertEqual(extended_block["times"], 3) self.assertEqual(test_block["times"], 3)
tests/test_runner.py
+252 -197
View File
@@ -1,8 +1,7 @@
import os import os
import time import time
from httprunner import exceptions, loader, parser, runner from httprunner import loader, parser, runner
from httprunner.utils import deep_update_dict
from tests.api_server import HTTPBIN_SERVER from tests.api_server import HTTPBIN_SERVER
from tests.base import ApiServerUnittest from tests.base import ApiServerUnittest
@@ -19,7 +18,7 @@ class TestRunner(ApiServerUnittest):
"base_url": "http://127.0.0.1", "base_url": "http://127.0.0.1",
"verify": False "verify": False
} }
self.test_runner = runner.Runner(config, self.debugtalk_functions) self.test_runner = runner.Runner(config)
self.reset_all() self.reset_all()
def reset_all(self): def reset_all(self):
@@ -36,214 +35,253 @@ class TestRunner(ApiServerUnittest):
] ]
for testcase_file_path in testcase_file_path_list: for testcase_file_path in testcase_file_path_list:
testcases = loader.load_file(testcase_file_path) tests_mapping = loader.load_tests(testcase_file_path)
testcases = parser.prepare_lazy_data( parsed_tests_mapping = parser.parse_tests(tests_mapping)
testcases, parsed_testcase = parsed_tests_mapping["testcases"][0]
self.debugtalk_functions, test_runner = runner.Runner(parsed_testcase["config"])
{"expect_status_code", "token_len", "token", "success"} test_runner.run_test(parsed_testcase["teststeps"][0])
) test_runner.run_test(parsed_testcase["teststeps"][1])
config_dict = {} test_runner.run_test(parsed_testcase["teststeps"][2])
test_runner = runner.Runner(config_dict, self.debugtalk_functions)
test = testcases[0]["test"]
test_runner.run_test(test)
test = testcases[1]["test"]
test_runner.run_test(test)
test = testcases[2]["test"]
test_runner.run_test(test)
def test_run_single_testcase_fail(self):
test = {
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 205},
{"check": "content.token", "comparator": "len_eq", "expect": 19}
]
}
with self.assertRaises(exceptions.ValidationFailure):
self.test_runner.run_test(test)
def test_run_testcase_with_hooks(self): def test_run_testcase_with_hooks(self):
start_time = time.time() start_time = time.time()
config_dict = { testcases = [
"name": "basic test with httpbin", {
"base_url": HTTPBIN_SERVER, "config": {
"setup_hooks": [ "name": "basic test with httpbin",
"${sleep_N_secs(0.5)}", "base_url": HTTPBIN_SERVER,
"${hook_print(setup)}" "setup_hooks": [
], "${sleep_N_secs(0.5)}",
"teardown_hooks": [ "${hook_print(setup)}"
"${sleep_N_secs(1)}", ],
"${hook_print(teardown)}" "teardown_hooks": [
] "${sleep_N_secs(1)}",
"${hook_print(teardown)}"
]
},
"teststeps": [
{
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 200}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"testcases": testcases
} }
prepared_config_dict = parser.prepare_lazy_data(config_dict, self.debugtalk_functions) parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_runner = runner.Runner(prepared_config_dict, self.debugtalk_functions) parsed_testcase = parsed_tests_mapping["testcases"][0]
test_runner = runner.Runner(parsed_testcase["config"])
end_time = time.time() end_time = time.time()
# check if testcase setup hook executed # check if testcase setup hook executed
self.assertGreater(end_time - start_time, 0.5) self.assertGreater(end_time - start_time, 0.5)
start_time = time.time() start_time = time.time()
test = { test_runner.run_test(parsed_testcase["teststeps"][0])
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 200}
]
}
test_runner.run_test(test)
test_runner.run_test(test)
end_time = time.time() end_time = time.time()
# testcase teardown hook has not been executed now # testcase teardown hook has not been executed now
self.assertLess(end_time - start_time, 1) self.assertLess(end_time - start_time, 1)
def test_run_testcase_with_hooks_assignment(self): def test_run_testcase_with_hooks_assignment(self):
config_dict = { testcases = [
"name": "basic test with httpbin", {
"base_url": HTTPBIN_SERVER "config": {
} "name": "basic test with httpbin",
test = { "base_url": HTTPBIN_SERVER
"name": "modify request headers",
"base_url": HTTPBIN_SERVER,
"request": {
"url": "/anything",
"method": "POST",
"headers": {
"user_agent": "iOS/10.3",
"os_platform": "ios"
}, },
"data": "a=1&b=2" "teststeps": [
{
"name": "modify request headers",
"base_url": HTTPBIN_SERVER,
"request": {
"url": "/anything",
"method": "POST",
"headers": {
"user_agent": "iOS/10.3",
"os_platform": "ios"
},
"data": "a=1&b=2"
},
"setup_hooks": [
{"total": "${sum_two(1, 5)}"}
],
"validate": [
{"check": "status_code", "expect": 200}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
}, },
"setup_hooks": [ "testcases": testcases
{"total": "${sum_two(1, 5)}"}
],
"validate": [
{"check": "status_code", "expect": 200}
]
} }
parsed_test = parser.prepare_lazy_data(test, self.debugtalk_functions) parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_runner = runner.Runner(config_dict, self.debugtalk_functions) parsed_testcase = parsed_tests_mapping["testcases"][0]
test_runner.run_test(parsed_test) test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
test_variables_mapping = test_runner.session_context.test_variables_mapping test_variables_mapping = test_runner.session_context.test_variables_mapping
self.assertEqual(test_variables_mapping["total"], 6) self.assertEqual(test_variables_mapping["total"], 6)
self.assertEqual(test_variables_mapping["request"]["data"], "a=1&b=2") self.assertEqual(test_variables_mapping["request"]["data"], "a=1&b=2")
def test_run_testcase_with_hooks_modify_request(self): def test_run_testcase_with_hooks_modify_request(self):
config_dict = { testcases = [
"name": "basic test with httpbin", {
"base_url": HTTPBIN_SERVER "config": {
} "name": "basic test with httpbin",
test = { "base_url": HTTPBIN_SERVER
"name": "modify request headers",
"base_url": HTTPBIN_SERVER,
"request": {
"url": "/anything",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3"
}, },
"json": { "teststeps": [
"os_platform": "ios", {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" "name": "modify request headers",
} "base_url": HTTPBIN_SERVER,
"request": {
"url": "/anything",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3"
},
"json": {
"os_platform": "ios",
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"setup_hooks": [
"${modify_request_json($request, android)}"
],
"validate": [
{"check": "status_code", "expect": 200},
{"check": "content.json.os_platform", "expect": "android"}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
}, },
"setup_hooks": [ "testcases": testcases
"${modify_request_json($request, android)}"
],
"validate": [
{"check": "status_code", "expect": 200},
{"check": "content.json.os_platform", "expect": "android"}
]
} }
test_runner = runner.Runner(config_dict, self.debugtalk_functions) parsed_tests_mapping = parser.parse_tests(tests_mapping)
parsed_test = parser.prepare_lazy_data(test, self.debugtalk_functions, {"request"}) parsed_testcase = parsed_tests_mapping["testcases"][0]
test_runner.run_test(parsed_test) test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
def test_run_testcase_with_teardown_hooks_success(self): def test_run_testcase_with_teardown_hooks_success(self):
test = { testcases = [
"name": "get token", {
"request": { "config": {
"url": "http://127.0.0.1:5000/api/get-token", "name": "basic test with httpbin"
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
}, },
"json": { "teststeps": [
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" {
} "name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 200}
],
"teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
}, },
"validate": [ "testcases": testcases
{"check": "status_code", "expect": 200}
],
"teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"]
} }
parsed_tests_mapping = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_tests_mapping["testcases"][0]
test_runner = runner.Runner(parsed_testcase["config"])
start_time = time.time() start_time = time.time()
self.test_runner.run_test(test) test_runner.run_test(parsed_testcase["teststeps"][0])
end_time = time.time() end_time = time.time()
# check if teardown function executed # check if teardown function executed
self.assertLess(end_time - start_time, 0.5) self.assertLess(end_time - start_time, 0.5)
def test_run_testcase_with_teardown_hooks_fail(self): def test_run_testcase_with_teardown_hooks_fail(self):
test = { testcases = [
"name": "get token", {
"request": { "config": {
"url": "http://127.0.0.1:5000/api/get-token2", "name": "basic test with httpbin"
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
}, },
"json": { "teststeps": [
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" {
} "name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token2",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 404}
],
"teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
}, },
"validate": [ "testcases": testcases
{"check": "status_code", "expect": 404}
],
"teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"]
} }
prepared_test = parser.prepare_lazy_data(test, self.debugtalk_functions, {"response"}) parsed_tests_mapping = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_tests_mapping["testcases"][0]
test_runner = runner.Runner(parsed_testcase["config"])
start_time = time.time() start_time = time.time()
self.test_runner.run_test(prepared_test) test_runner.run_test(parsed_testcase["teststeps"][0])
end_time = time.time() end_time = time.time()
# check if teardown function executed # check if teardown function executed
self.assertGreater(end_time - start_time, 2) self.assertGreater(end_time - start_time, 2)
@@ -251,34 +289,51 @@ class TestRunner(ApiServerUnittest):
def test_bugfix_type_match(self): def test_bugfix_type_match(self):
testcase_file_path = os.path.join( testcase_file_path = os.path.join(
os.getcwd(), 'tests/data/bugfix_type_match.yml') os.getcwd(), 'tests/data/bugfix_type_match.yml')
testcases = loader.load_file(testcase_file_path) tests_mapping = loader.load_tests(testcase_file_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test = testcases[1]["test"] parsed_testcase = parsed_tests_mapping["testcases"][0]
self.test_runner.run_test(test) test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
def test_run_validate_elapsed(self): def test_run_validate_elapsed(self):
test = { testcases = [
"name": "get token", {
"request": { "config": {},
"url": "http://127.0.0.1:5000/api/get-token", "teststeps": [
"method": "POST", {
"headers": { "name": "get token",
"content-type": "application/json", "request": {
"user_agent": "iOS/10.3", "url": "http://127.0.0.1:5000/api/get-token",
"device_sn": "HZfFBh6tU59EdXJ", "method": "POST",
"os_platform": "ios", "headers": {
"app_version": "2.8.6" "content-type": "application/json",
}, "user_agent": "iOS/10.3",
"json": { "device_sn": "HZfFBh6tU59EdXJ",
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" "os_platform": "ios",
} "app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 200},
{"check": "elapsed.seconds", "comparator": "lt", "expect": 1},
{"check": "elapsed.days", "comparator": "eq", "expect": 0},
{"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000},
{"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
}, },
"validate": [ "testcases": testcases
{"check": "status_code", "expect": 200},
{"check": "elapsed.seconds", "comparator": "lt", "expect": 1},
{"check": "elapsed.days", "comparator": "eq", "expect": 0},
{"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000},
{"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1}
]
} }
self.test_runner.run_test(test) parsed_tests_mapping = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_tests_mapping["testcases"][0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
tests/test_utils.py
-84
View File
@@ -61,35 +61,6 @@ class TestUtils(ApiServerUnittest):
result = utils.query_json(json_content, query) result = utils.query_json(json_content, query)
self.assertEqual(result, "L") self.assertEqual(result, "L")
def test_get_uniform_comparator(self):
self.assertEqual(utils.get_uniform_comparator("eq"), "equals")
self.assertEqual(utils.get_uniform_comparator("=="), "equals")
self.assertEqual(utils.get_uniform_comparator("lt"), "less_than")
self.assertEqual(utils.get_uniform_comparator("le"), "less_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("gt"), "greater_than")
self.assertEqual(utils.get_uniform_comparator("ge"), "greater_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("ne"), "not_equals")
self.assertEqual(utils.get_uniform_comparator("str_eq"), "string_equals")
self.assertEqual(utils.get_uniform_comparator("len_eq"), "length_equals")
self.assertEqual(utils.get_uniform_comparator("count_eq"), "length_equals")
self.assertEqual(utils.get_uniform_comparator("len_gt"), "length_greater_than")
self.assertEqual(utils.get_uniform_comparator("count_gt"), "length_greater_than")
self.assertEqual(utils.get_uniform_comparator("count_greater_than"), "length_greater_than")
self.assertEqual(utils.get_uniform_comparator("len_ge"), "length_greater_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("count_ge"), "length_greater_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("count_greater_than_or_equals"), "length_greater_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("len_lt"), "length_less_than")
self.assertEqual(utils.get_uniform_comparator("count_lt"), "length_less_than")
self.assertEqual(utils.get_uniform_comparator("count_less_than"), "length_less_than")
self.assertEqual(utils.get_uniform_comparator("len_le"), "length_less_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("count_le"), "length_less_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("count_less_than_or_equals"), "length_less_than_or_equals")
def current_validators(self): def current_validators(self):
from httprunner import built_in from httprunner import built_in
functions_mapping = loader.load_module_functions(built_in) functions_mapping = loader.load_module_functions(built_in)
@@ -205,61 +176,6 @@ class TestUtils(ApiServerUnittest):
self.assertIsInstance(ordered_dict, dict) self.assertIsInstance(ordered_dict, dict)
self.assertIn("a", ordered_dict) self.assertIn("a", ordered_dict)
def test_extend_validators(self):
def_validators = [
{'eq': ['v1', 200]},
{"check": "s2", "expect": 16, "comparator": "len_eq"}
]
current_validators = [
{"check": "v1", "expect": 201},
{'len_eq': ['s3', 12]}
]
def_validators = [
parser.parse_validator(validator)
for validator in def_validators
]
ref_validators = [
parser.parse_validator(validator)
for validator in current_validators
]
extended_validators = utils.extend_validators(def_validators, ref_validators)
self.assertIn(
{"check": "v1", "expect": 201, "comparator": "eq"},
extended_validators
)
self.assertIn(
{"check": "s2", "expect": 16, "comparator": "len_eq"},
extended_validators
)
self.assertIn(
{"check": "s3", "expect": 12, "comparator": "len_eq"},
extended_validators
)
def test_extend_validators_with_dict(self):
def_validators = [
{'eq': ["a", {"v": 1}]},
{'eq': [{"b": 1}, 200]}
]
current_validators = [
{'len_eq': ['s3', 12]},
{'eq': [{"b": 1}, 201]}
]
def_validators = [
parser.parse_validator(validator)
for validator in def_validators
]
ref_validators = [
parser.parse_validator(validator)
for validator in current_validators
]
extended_validators = utils.extend_validators(def_validators, ref_validators)
self.assertEqual(len(extended_validators), 3)
self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'eq'}, extended_validators)
self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'eq'}, extended_validators)
def test_extend_variables(self): def test_extend_variables(self):
raw_variables = [{"var1": "val1"}, {"var2": "val2"}] raw_variables = [{"var1": "val1"}, {"var2": "val2"}]
override_variables = [{"var1": "val111"}, {"var3": "val3"}] override_variables = [{"var1": "val111"}, {"var3": "val3"}]
tests/test_validator.py
+98
View File
@@ -74,3 +74,101 @@ class TestValidator(unittest.TestCase):
func = lambda x: x + 1 func = lambda x: x + 1
self.assertTrue(validator.is_function(func)) self.assertTrue(validator.is_function(func))
self.assertTrue(validator.is_function(validator.is_testcase)) self.assertTrue(validator.is_function(validator.is_testcase))
def test_get_uniform_comparator(self):
self.assertEqual(validator.get_uniform_comparator("eq"), "equals")
self.assertEqual(validator.get_uniform_comparator("=="), "equals")
self.assertEqual(validator.get_uniform_comparator("lt"), "less_than")
self.assertEqual(validator.get_uniform_comparator("le"), "less_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("gt"), "greater_than")
self.assertEqual(validator.get_uniform_comparator("ge"), "greater_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("ne"), "not_equals")
self.assertEqual(validator.get_uniform_comparator("str_eq"), "string_equals")
self.assertEqual(validator.get_uniform_comparator("len_eq"), "length_equals")
self.assertEqual(validator.get_uniform_comparator("count_eq"), "length_equals")
self.assertEqual(validator.get_uniform_comparator("len_gt"), "length_greater_than")
self.assertEqual(validator.get_uniform_comparator("count_gt"), "length_greater_than")
self.assertEqual(validator.get_uniform_comparator("count_greater_than"), "length_greater_than")
self.assertEqual(validator.get_uniform_comparator("len_ge"), "length_greater_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("count_ge"), "length_greater_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("count_greater_than_or_equals"), "length_greater_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("len_lt"), "length_less_than")
self.assertEqual(validator.get_uniform_comparator("count_lt"), "length_less_than")
self.assertEqual(validator.get_uniform_comparator("count_less_than"), "length_less_than")
self.assertEqual(validator.get_uniform_comparator("len_le"), "length_less_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("count_le"), "length_less_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("count_less_than_or_equals"), "length_less_than_or_equals")
def test_parse_validator(self):
_validator = {"check": "status_code", "comparator": "eq", "expect": 201}
self.assertEqual(
validator.uniform_validator(_validator),
{"check": "status_code", "comparator": "equals", "expect": 201}
)
_validator = {'eq': ['status_code', 201]}
self.assertEqual(
validator.uniform_validator(_validator),
{"check": "status_code", "comparator": "equals", "expect": 201}
)
def test_extend_validators(self):
def_validators = [
{'eq': ['v1', 200]},
{"check": "s2", "expect": 16, "comparator": "len_eq"}
]
current_validators = [
{"check": "v1", "expect": 201},
{'len_eq': ['s3', 12]}
]
def_validators = [
validator.uniform_validator(_validator)
for _validator in def_validators
]
ref_validators = [
validator.uniform_validator(_validator)
for _validator in current_validators
]
extended_validators = validator.extend_validators(def_validators, ref_validators)
self.assertIn(
{"check": "v1", "expect": 201, "comparator": "equals"},
extended_validators
)
self.assertIn(
{"check": "s2", "expect": 16, "comparator": "length_equals"},
extended_validators
)
self.assertIn(
{"check": "s3", "expect": 12, "comparator": "length_equals"},
extended_validators
)
def test_extend_validators_with_dict(self):
def_validators = [
{'eq': ["a", {"v": 1}]},
{'eq': [{"b": 1}, 200]}
]
current_validators = [
{'len_eq': ['s3', 12]},
{'eq': [{"b": 1}, 201]}
]
def_validators = [
validator.uniform_validator(_validator)
for _validator in def_validators
]
ref_validators = [
validator.uniform_validator(_validator)
for _validator in current_validators
]
extended_validators = validator.extend_validators(def_validators, ref_validators)
self.assertEqual(len(extended_validators), 3)
self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'equals'}, extended_validators)
self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'equals'}, extended_validators)