GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-05-13 08:59:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #555 from HttpRunner/lazy_parser

Browse Source 
2.1.0 Lazy parser
This commit is contained in:
debugtalk
2019-04-10 20:48:52 +08:00
committed by GitHub
parent f60681d668 8cd5ac4b25
commit 1bd99668cb
36 changed files with 1894 additions and 1491 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
HISTORY.md
View File

@@ -1,5 +1,11 @@
# Release History
## 2.1.0 (2019-03-19)
**Features**
- implement json dump Python objects when save tests
## 2.0.6 (2019-03-18)
**Features**

2
httprunner/__about__.py
View File

@@ -1,7 +1,7 @@
__title__ = 'HttpRunner'
__description__ = 'One-stop solution for HTTP(S) testing.'
__url__ = 'https://github.com/HttpRunner/HttpRunner'
__version__ = '2.0.6'
__version__ = '2.1.0'
__author__ = 'debugtalk'
__author_email__ = 'mail@debugtalk.com'
__license__ = 'Apache-2.0'

53
httprunner/api.py
View File

@@ -36,11 +36,11 @@ class HttpRunner(object):
if log_file:
logger.setup_logger(log_level, log_file)
def _add_tests(self, tests_mapping):
def _add_tests(self, testcases):
""" initialize testcase with Runner() and add to test suite.
Args:
tests_mapping (dict): project info and testcases list.
testcases (list): testcases list.
Returns:
unittest.TestSuite()
@@ -60,18 +60,23 @@ class HttpRunner(object):
if "config" in test_dict:
# run nested testcase
test.__doc__ = test_dict["config"].get("name")
variables = test_dict["config"].get("variables", {})
else:
# run api test
test.__doc__ = test_dict.get("name")
variables = test_dict.get("variables", {})
if isinstance(test.__doc__, parser.LazyString):
parsed_variables = parser.parse_variables_mapping(variables, ignore=True)
test.__doc__ = parser.parse_lazy_data(
test.__doc__, parsed_variables)
return test
test_suite = unittest.TestSuite()
functions = tests_mapping.get("project_mapping", {}).get("functions", {})
for testcase in tests_mapping["testcases"]:
for testcase in testcases:
config = testcase.get("config", {})
test_runner = runner.Runner(config, functions)
test_runner = runner.Runner(config)
TestSequense = type('TestSequense', (unittest.TestCase,), {})
tests = testcase.get("teststeps", [])
@@ -157,19 +162,20 @@ class HttpRunner(object):
def run_tests(self, tests_mapping):
""" run testcase/testsuite data
"""
project_mapping = tests_mapping.get("project_mapping", {})
if self.save_tests:
utils.dump_tests(tests_mapping, "loaded")
utils.dump_logs(tests_mapping, project_mapping, "loaded")
# parse tests
self.exception_stage = "parse tests"
parsed_tests_mapping = parser.parse_tests(tests_mapping)
parsed_testcases = parser.parse_tests(tests_mapping)
if self.save_tests:
utils.dump_tests(parsed_tests_mapping, "parsed")
utils.dump_logs(parsed_testcases, project_mapping, "parsed")
# add tests to test suite
self.exception_stage = "add tests to test suite"
test_suite = self._add_tests(parsed_tests_mapping)
test_suite = self._add_tests(parsed_testcases)
# run test suite
self.exception_stage = "run test suite"
@@ -184,7 +190,7 @@ class HttpRunner(object):
report.stringify_summary(self._summary)
if self.save_tests:
utils.dump_summary(self._summary, tests_mapping["project_mapping"])
utils.dump_logs(self._summary, project_mapping, "summary")
report_path = report.render_html_report(
self._summary,
@@ -275,27 +281,22 @@ def prepare_locust_tests(path):
path (str): testcase file path.
Returns:
dict: locust tests data
list: locust tests data
{
"functions": {},
"tests": []
}
[
testcase1_dict,
testcase2_dict
]
"""
tests_mapping = loader.load_tests(path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
testcases = parser.parse_tests(tests_mapping)
functions = parsed_tests_mapping.get("project_mapping", {}).get("functions", {})
locust_tests = []
tests = []
for testcase in parsed_tests_mapping["testcases"]:
for testcase in testcases:
testcase_weight = testcase.get("config", {}).pop("weight", 1)
for _ in range(testcase_weight):
tests.append(testcase)
locust_tests.append(testcase)
return {
"functions": functions,
"tests": tests
}
return locust_tests

3
httprunner/cli.py
View File

@@ -8,8 +8,9 @@ def main_hrun():
from httprunner.__about__ import __description__, __version__
from httprunner.api import HttpRunner
from httprunner.compat import is_py2
from httprunner.validator import validate_json_file
from httprunner.utils import (create_scaffold, get_python2_retire_msg,
prettify_json_file, validate_json_file)
prettify_json_file)
parser = argparse.ArgumentParser(description=__description__)
parser.add_argument(

14
httprunner/client.py
View File

@@ -5,7 +5,7 @@ import time
import requests
import urllib3
from httprunner import logger
from httprunner.utils import build_url, lower_dict_keys, omit_long_data
from httprunner.utils import lower_dict_keys, omit_long_data
from requests import Request, Response
from requests.exceptions import (InvalidSchema, InvalidURL, MissingSchema,
RequestException)
@@ -28,15 +28,10 @@ class HttpSession(requests.Session):
display statistics.
This is a slightly extended version of `python-request <http://python-requests.org>`_'s
:py:class:`requests.Session` class and mostly this class works exactly the same. However
the methods for making requests (get, post, delete, put, head, options, patch, request)
can now take a *url* argument that's only the path part of the URL, in which case the host
part of the URL will be prepended with the HttpSession.base_url which is normally inherited
from a HttpRunner class' host property.
:py:class:`requests.Session` class and mostly this class works exactly the same.
"""
def __init__(self, base_url=None, *args, **kwargs):
def __init__(self, *args, **kwargs):
super(HttpSession, self).__init__(*args, **kwargs)
self.base_url = base_url if base_url else ""
self.init_meta_data()
def init_meta_data(self):
@@ -180,9 +175,6 @@ class HttpSession(requests.Session):
kwargs.setdefault("timeout", 120)
self.meta_data["data"][0]["request"].update(kwargs)
# prepend url with hostname unless it's already an absolute URL
url = build_url(self.base_url, url)
start_timestamp = time.time()
response = self._send_request_safe_mode(method, url, **kwargs)
response_time_ms = round((time.time() - start_timestamp) * 1000, 2)

175
httprunner/context.py
View File

@@ -5,18 +5,17 @@ class SessionContext(object):
""" HttpRunner session, store runtime variables.
Examples:
>>> functions={...}
>>> variables = {"SECRET_KEY": "DebugTalk"}
>>> context = SessionContext(functions, variables)
>>> context = SessionContext(variables)
Equivalent to:
>>> context = SessionContext(functions)
>>> context = SessionContext()
>>> context.update_session_variables(variables)
"""
def __init__(self, functions, variables=None):
self.session_variables_mapping = utils.ensure_mapping_format(variables or {})
self.FUNCTIONS_MAPPING = functions
def __init__(self, variables=None):
variables_mapping = utils.ensure_mapping_format(variables or {})
self.session_variables_mapping = parser.parse_variables_mapping(variables_mapping)
self.init_test_variables()
self.validation_results = []
@@ -36,16 +35,14 @@ class SessionContext(object):
"""
variables_mapping = variables_mapping or {}
variables_mapping = utils.ensure_mapping_format(variables_mapping)
variables_mapping.update(self.session_variables_mapping)
parsed_variables_mapping = parser.parse_variables_mapping(variables_mapping)
self.test_variables_mapping = {}
# priority: extracted variable > teststep variable
self.test_variables_mapping.update(variables_mapping)
self.test_variables_mapping.update(parsed_variables_mapping)
self.test_variables_mapping.update(self.session_variables_mapping)
for variable_name, variable_value in variables_mapping.items():
variable_value = self.eval_content(variable_value)
self.update_test_variables(variable_name, variable_value)
def update_test_variables(self, variable_name, variable_value):
""" update test variables, these variables are only valid in the current test.
"""
@@ -63,110 +60,46 @@ class SessionContext(object):
""" evaluate content recursively, take effect on each variable and function in content.
content may be in any data structure, include dict, list, tuple, number, string, etc.
"""
return parser.parse_data(
content,
self.test_variables_mapping,
self.FUNCTIONS_MAPPING
)
return parser.parse_lazy_data(content, self.test_variables_mapping)
def __eval_check_item(self, validator, resp_obj):
def __eval_validator_check(self, check_item, resp_obj):
""" evaluate check item in validator.
Args:
validator (dict): validator
{"check": "status_code", "comparator": "eq", "expect": 201}
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
resp_obj (object): requests.Response() object
check_item: check_item should only be the following 5 formats:
1, variable reference, e.g. $token
2, function reference, e.g. ${is_status_code_200($status_code)}
3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
4, string joined by delimiter. e.g. "status_code", "headers.content-type"
5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
Returns:
dict: validator info
{
"check": "status_code",
"check_value": 200,
"expect": 201,
"comparator": "eq"
}
resp_obj: response object
"""
check_item = validator["check"]
# check_item should only be the following 5 formats:
# 1, variable reference, e.g. $token
# 2, function reference, e.g. ${is_status_code_200($status_code)}
# 3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
# 4, string joined by delimiter. e.g. "status_code", "headers.content-type"
# 5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
if isinstance(check_item, (dict, list)) \
or parser.extract_variables(check_item) \
or parser.extract_functions(check_item):
or isinstance(check_item, parser.LazyString):
# format 1/2/3
check_value = self.eval_content(check_item)
else:
# format 4/5
check_value = resp_obj.extract_field(check_item)
validator["check_value"] = check_value
return check_value
# expect_value should only be in 2 types:
# 1, variable reference, e.g. $expect_status_code
# 2, actual value, e.g. 200
expect_value = self.eval_content(validator["expect"])
validator["expect"] = expect_value
validator["check_result"] = "unchecked"
return validator
def _do_validation(self, validator_dict):
""" validate with functions
def __eval_validator_expect(self, expect_item):
""" evaluate expect item in validator.
Args:
validator_dict (dict): validator dict
{
"check": "status_code",
"check_value": 200,
"expect": 201,
"comparator": "eq"
}
expect_item: expect_item should only be in 2 types:
1, variable reference, e.g. $expect_status_code
2, actual value, e.g. 200
"""
# TODO: move comparator uniform to init_test_suites
comparator = utils.get_uniform_comparator(validator_dict["comparator"])
validate_func = parser.get_mapping_function(comparator, self.FUNCTIONS_MAPPING)
check_item = validator_dict["check"]
check_value = validator_dict["check_value"]
expect_value = validator_dict["expect"]
if (check_value is None or expect_value is None) \
and comparator not in ["is", "eq", "equals", "=="]:
raise exceptions.ParamsError("Null value can only be compared with comparator: eq/equals/==")
validate_msg = "validate: {} {} {}({})".format(
check_item,
comparator,
expect_value,
type(expect_value).__name__
)
try:
validator_dict["check_result"] = "pass"
validate_func(check_value, expect_value)
validate_msg += "\t==> pass"
logger.log_debug(validate_msg)
except (AssertionError, TypeError):
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__,
comparator,
expect_value,
type(expect_value).__name__
)
logger.log_error(validate_msg)
validator_dict["check_result"] = "fail"
raise exceptions.ValidationFailure(validate_msg)
expect_value = self.eval_content(expect_item)
return expect_value
def validate(self, validators, resp_obj):
""" make validations
""" make validation with comparators
"""
self.validation_results = []
if not validators:
@@ -178,19 +111,59 @@ class SessionContext(object):
failures = []
for validator in validators:
# evaluate validators with context variable mapping.
evaluated_validator = self.__eval_check_item(
parser.parse_validator(validator),
# validator should be LazyFunction object
if not isinstance(validator, parser.LazyFunction):
raise exceptions.ValidationFailure(
"validator should be parsed first: {}".format(validators))
# evaluate validator args with context variable mapping.
validator_args = validator.get_args()
check_item, expect_item = validator_args
check_value = self.__eval_validator_check(
check_item,
resp_obj
)
expect_value = self.__eval_validator_expect(expect_item)
validator.update_args([check_value, expect_value])
comparator = validator.func_name
validator_dict = {
"comparator": comparator,
"check": check_item,
"check_value": check_value,
"expect": expect_item,
"expect_value": expect_value
}
validate_msg = "\nvalidate: {} {} {}({})".format(
check_item,
comparator,
expect_value,
type(expect_value).__name__
)
try:
self._do_validation(evaluated_validator)
except exceptions.ValidationFailure as ex:
validator.to_value(self.test_variables_mapping)
validator_dict["check_result"] = "pass"
validate_msg += "\t==> pass"
logger.log_debug(validate_msg)
except (AssertionError, TypeError):
validate_pass = False
failures.append(str(ex))
validator_dict["check_result"] = "fail"
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__,
comparator,
expect_value,
type(expect_value).__name__
)
logger.log_error(validate_msg)
failures.append(validate_msg)
self.validation_results.append(evaluated_validator)
self.validation_results.append(validator_dict)
# restore validator args, in case of running multiple times
validator.update_args(validator_args)
if not validate_pass:
failures_string = "\n".join([failure for failure in failures])

1086
httprunner/parser.py
View File

File diff suppressed because it is too large Load Diff

72
httprunner/runner.py
View File

@@ -11,27 +11,40 @@ class Runner(object):
""" Running testcases.
Examples:
>>> functions={...}
>>> config = {
"name": "XXXX",
"base_url": "http://127.0.0.1",
"verify": False
>>> tests_mapping = {
"project_mapping": {
"functions": {}
},
"testcases": [
{
"config": {
"name": "XXXX",
"base_url": "http://127.0.0.1",
"verify": False
},
"teststeps": [
{
"name": "test description",
"variables": [], # optional
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "GET"
}
}
]
}
]
}
>>> runner = Runner(config, functions)
>>> test_dict = {
"name": "test description",
"variables": [], # optional
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "GET"
}
}
>>> runner.run_test(test_dict)
>>> testcases = parser.parse_tests(tests_mapping)
>>> parsed_testcase = testcases[0]
>>> test_runner = runner.Runner(parsed_testcase["config"])
>>> test_runner.run_test(parsed_testcase["teststeps"][0])
"""
def __init__(self, config, functions, http_client_session=None):
def __init__(self, config, http_client_session=None):
""" run testcase or testsuite.
Args:
@@ -47,19 +60,18 @@ class Runner(object):
http_client_session (instance): requests.Session(), or locust.client.Session() instance.
"""
base_url = config.get("base_url")
self.verify = config.get("verify", True)
self.output = config.get("output", [])
self.functions = functions
self.validation_results = []
config_variables = config.get("variables", {})
# testcase setup hooks
testcase_setup_hooks = config.get("setup_hooks", [])
# testcase teardown hooks
self.testcase_teardown_hooks = config.get("teardown_hooks", [])
self.http_client_session = http_client_session or HttpSession(base_url)
self.session_context = SessionContext(self.functions)
self.http_client_session = http_client_session or HttpSession()
self.session_context = SessionContext(config_variables)
if testcase_setup_hooks:
self.do_hook_actions(testcase_setup_hooks, "setup")
@@ -199,20 +211,24 @@ class Runner(object):
self.session_context.init_test_variables(test_variables)
# teststep name
test_name = test_dict.get("name", "")
test_name = self.session_context.eval_content(test_dict.get("name", ""))
# parse test request
raw_request = test_dict.get('request', {})
parsed_test_request = self.session_context.eval_content(raw_request)
self.session_context.update_test_variables("request", parsed_test_request)
# prepend url with base_url unless it's already an absolute URL
url = parsed_test_request.pop('url')
base_url = self.session_context.eval_content(test_dict.get("base_url", ""))
parsed_url = utils.build_url(base_url, url)
# setup hooks
setup_hooks = test_dict.get("setup_hooks", [])
if setup_hooks:
self.do_hook_actions(setup_hooks, "setup")
try:
url = parsed_test_request.pop('url')
method = parsed_test_request.pop('method')
parsed_test_request.setdefault("verify", self.verify)
group_name = parsed_test_request.pop("group", None)
@@ -227,13 +243,13 @@ class Runner(object):
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
logger.log_info("{method} {url}".format(method=method, url=url))
logger.log_info("{method} {url}".format(method=method, url=parsed_url))
logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
# request
resp = self.http_client_session.request(
method,
url,
parsed_url,
name=(group_name or test_name),
**parsed_test_request
)
@@ -254,13 +270,12 @@ class Runner(object):
validators = test_dict.get("validate", [])
try:
self.session_context.validate(validators, resp_obj)
except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure):
err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32)
# log request
err_msg += "====== request details ======\n"
err_msg += "url: {}\n".format(url)
err_msg += "url: {}\n".format(parsed_url)
err_msg += "method: {}\n".format(method)
err_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {}))
for k, v in parsed_test_request.items():
@@ -288,7 +303,7 @@ class Runner(object):
config = testcase_dict.get("config", {})
# each teststeps in one testcase (YAML/JSON) share the same session.
test_runner = Runner(config, self.functions, self.http_client_session)
test_runner = Runner(config, self.http_client_session)
tests = testcase_dict.get("teststeps", [])
@@ -351,6 +366,9 @@ class Runner(object):
self.meta_datas = None
if "teststeps" in test_dict:
# nested testcase
test_dict.setdefault("config", {}).setdefault("variables", {})
test_dict["config"]["variables"].update(
self.session_context.session_variables_mapping)
self._run_testcase(test_dict)
else:
# api

11
httprunner/templates/locustfile_template
View File

@@ -15,7 +15,8 @@ logging.getLogger('locust.runners').setLevel(logging.INFO)
class WebPageTasks(TaskSet):
def on_start(self):
self.test_runner = Runner(self.locust.config, self.locust.functions, self.client)
config = {}
self.test_runner = Runner(config, self.client)
@task
def test_any(self):
@@ -32,14 +33,10 @@ class WebPageTasks(TaskSet):
class WebPageUser(HttpLocust):
host = ""
task_set = WebPageTasks
min_wait = 10
max_wait = 30
file_path = "$TESTCASE_FILE"
locust_tests = prepare_locust_tests(file_path)
functions = locust_tests["functions"]
tests = locust_tests["tests"]
config = {}
host = config.get('base_url', '')
tests = prepare_locust_tests(file_path)

184
httprunner/utils.py
View File

@@ -53,7 +53,7 @@ def get_os_environ(variable_name):
def build_url(base_url, path):
""" prepend url with hostname unless it's already an absolute URL """
""" prepend url with base_url unless it's already an absolute URL """
if absolute_http_url_regexp.match(path):
return path
elif base_url:
@@ -120,39 +120,6 @@ def query_json(json_content, query, delimiter='.'):
return json_content
def get_uniform_comparator(comparator):
""" convert comparator alias to uniform name
"""
if comparator in ["eq", "equals", "==", "is"]:
return "equals"
elif comparator in ["lt", "less_than"]:
return "less_than"
elif comparator in ["le", "less_than_or_equals"]:
return "less_than_or_equals"
elif comparator in ["gt", "greater_than"]:
return "greater_than"
elif comparator in ["ge", "greater_than_or_equals"]:
return "greater_than_or_equals"
elif comparator in ["ne", "not_equals"]:
return "not_equals"
elif comparator in ["str_eq", "string_equals"]:
return "string_equals"
elif comparator in ["len_eq", "length_equals", "count_eq"]:
return "length_equals"
elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]:
return "length_greater_than"
elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \
"count_greater_than_or_equals"]:
return "length_greater_than_or_equals"
elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]:
return "length_less_than"
elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \
"count_less_than_or_equals"]:
return "length_less_than_or_equals"
else:
return comparator
def deep_update_dict(origin_dict, override_dict):
""" update origin dict with override dict recursively
e.g. origin_dict = {'a': 1, 'b': {'c': 2, 'd': 4}}
@@ -323,78 +290,6 @@ def ensure_mapping_format(variables):
raise exceptions.ParamsError("variables format error!")
def _convert_validators_to_mapping(validators):
""" convert validators list to mapping.
Args:
validators (list): validators in list
Returns:
dict: validators mapping, use (check, comparator) as key.
Examples:
>>> validators = [
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": {"b": 1}, "expect": 200, "comparator": "eq"}
]
>>> _convert_validators_to_mapping(validators)
{
("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
}
"""
validators_mapping = {}
for validator in validators:
if not isinstance(validator["check"], collections.Hashable):
check = json.dumps(validator["check"])
else:
check = validator["check"]
key = (check, validator["comparator"])
validators_mapping[key] = validator
return validators_mapping
def extend_validators(raw_validators, override_validators):
""" extend raw_validators with override_validators.
override_validators will merge and override raw_validators.
Args:
raw_validators (dict):
override_validators (dict):
Returns:
list: extended validators
Examples:
>>> raw_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}]
>>> override_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}]
>>> extend_validators(raw_validators, override_validators)
[
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": "s2", "expect": 16, "comparator": "len_eq"},
{"check": "s3", "expect": 12, "comparator": "len_eq"}
]
"""
if not raw_validators:
return override_validators
elif not override_validators:
return raw_validators
else:
def_validators_mapping = _convert_validators_to_mapping(raw_validators)
ref_validators_mapping = _convert_validators_to_mapping(override_validators)
def_validators_mapping.update(ref_validators_mapping)
return list(def_validators_mapping.values())
def extend_variables(raw_variables, override_variables):
""" extend raw_variables with override_variables.
override_variables will merge and override raw_variables.
@@ -581,25 +476,6 @@ def gen_cartesian_product(*args):
return product_list
def validate_json_file(file_list):
""" validate JSON testcase format
"""
for json_file in set(file_list):
if not json_file.endswith(".json"):
logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file))
continue
logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN")
with io.open(json_file) as stream:
try:
json.load(stream)
except ValueError as e:
raise SystemExit(e)
print("OK")
def prettify_json_file(file_list):
""" prettify JSON testcase format
"""
@@ -649,6 +525,13 @@ def omit_long_data(body, omit_len=512):
def dump_json_file(json_data, pwd_dir_path, dump_file_name):
""" dump json data to file
"""
class PythonObjectEncoder(json.JSONEncoder):
def default(self, obj):
try:
return super().default(self, obj)
except TypeError:
return str(obj)
logs_dir_path = os.path.join(pwd_dir_path, "logs")
if not os.path.isdir(logs_dir_path):
os.makedirs(logs_dir_path)
@@ -663,7 +546,8 @@ def dump_json_file(json_data, pwd_dir_path, dump_file_name):
json_data,
indent=4,
separators=(',', ':'),
ensure_ascii=False
ensure_ascii=False,
cls=PythonObjectEncoder
))
)
else:
@@ -672,14 +556,15 @@ def dump_json_file(json_data, pwd_dir_path, dump_file_name):
outfile,
indent=4,
separators=(',', ':'),
ensure_ascii=False
ensure_ascii=False,
cls=PythonObjectEncoder
)
msg = "dump file: {}".format(dump_file_path)
logger.color_print(msg, "BLUE")
except TypeError:
msg = "Failed to dump json file: {}".format(dump_file_path)
except TypeError as ex:
msg = "Failed to dump json file: {}\nReason: {}".format(dump_file_path, ex)
logger.color_print(msg, "RED")
@@ -694,47 +579,18 @@ def _prepare_dump_info(project_mapping, tag_name):
return pwd_dir_path, dump_file_name
def dump_tests(tests_mapping, tag_name):
""" dump loaded/parsed tests data (except functions) to json file.
def dump_logs(json_data, project_mapping, tag_name):
""" dump tests data to json file.
the dumped file is located in PWD/logs folder.
Args:
tests_mapping (dict): data to dump
tag_name (str): tag name, loaded/parsed
json_data (list/dict): json data to dump
project_mapping (dict): project info
tag_name (str): tag name, loaded/parsed/summary
"""
project_mapping = tests_mapping.get("project_mapping", {})
pwd_dir_path, dump_file_name = _prepare_dump_info(project_mapping, tag_name)
tests_to_dump = {
"project_mapping": {}
}
for key in project_mapping:
if key != "functions":
tests_to_dump["project_mapping"][key] = project_mapping[key]
continue
# remove functions in order to dump
if project_mapping["functions"]:
debugtalk_py_path = os.path.join(pwd_dir_path, "debugtalk.py")
tests_to_dump["project_mapping"]["debugtalk.py"] = debugtalk_py_path
if "api" in tests_mapping:
tests_to_dump["api"] = tests_mapping["api"]
elif "testcases" in tests_mapping:
tests_to_dump["testcases"] = tests_mapping["testcases"]
elif "testsuites" in tests_mapping:
tests_to_dump["testsuites"] = tests_mapping["testsuites"]
dump_json_file(tests_to_dump, pwd_dir_path, dump_file_name)
def dump_summary(summary, project_mapping):
""" dump test result summary to json file.
"""
pwd_dir_path, dump_file_name = _prepare_dump_info(project_mapping, "summary")
dump_json_file(summary, pwd_dir_path, dump_file_name)
dump_json_file(json_data, pwd_dir_path, dump_file_name)
def get_python2_retire_msg():

188
httprunner/validator.py
View File

@@ -1,7 +1,12 @@
# encoding: utf-8
import collections
import io
import json
import os
import types
from httprunner import exceptions, logger
""" validate data format
TODO: refactor with JSON schema validate
@@ -129,6 +134,170 @@ def is_testcase_path(path):
return True
###############################################################################
## testcase validator utils
###############################################################################
def get_uniform_comparator(comparator):
""" convert comparator alias to uniform name
"""
if comparator in ["eq", "equals", "==", "is"]:
return "equals"
elif comparator in ["lt", "less_than"]:
return "less_than"
elif comparator in ["le", "less_than_or_equals"]:
return "less_than_or_equals"
elif comparator in ["gt", "greater_than"]:
return "greater_than"
elif comparator in ["ge", "greater_than_or_equals"]:
return "greater_than_or_equals"
elif comparator in ["ne", "not_equals"]:
return "not_equals"
elif comparator in ["str_eq", "string_equals"]:
return "string_equals"
elif comparator in ["len_eq", "length_equals", "count_eq"]:
return "length_equals"
elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]:
return "length_greater_than"
elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \
"count_greater_than_or_equals"]:
return "length_greater_than_or_equals"
elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]:
return "length_less_than"
elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \
"count_less_than_or_equals"]:
return "length_less_than_or_equals"
else:
return comparator
def uniform_validator(validator):
""" unify validator
Args:
validator (dict): validator maybe in two formats:
format1: this is kept for compatiblity with the previous versions.
{"check": "status_code", "comparator": "eq", "expect": 201}
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
format2: recommended new version, {comparator: [check_item, expected_value]}
{'eq': ['status_code', 201]}
{'eq': ['$resp_body_success', True]}
Returns
dict: validator info
{
"check": "status_code",
"expect": 201,
"comparator": "equals"
}
"""
if not isinstance(validator, dict):
raise exceptions.ParamsError("invalid validator: {}".format(validator))
if "check" in validator and "expect" in validator:
# format1
check_item = validator["check"]
expect_value = validator["expect"]
comparator = validator.get("comparator", "eq")
elif len(validator) == 1:
# format2
comparator = list(validator.keys())[0]
compare_values = validator[comparator]
if not isinstance(compare_values, list) or len(compare_values) != 2:
raise exceptions.ParamsError("invalid validator: {}".format(validator))
check_item, expect_value = compare_values
else:
raise exceptions.ParamsError("invalid validator: {}".format(validator))
# uniform comparator, e.g. lt => less_than, eq => equals
comparator = get_uniform_comparator(comparator)
return {
"check": check_item,
"expect": expect_value,
"comparator": comparator
}
def _convert_validators_to_mapping(validators):
""" convert validators list to mapping.
Args:
validators (list): validators in list
Returns:
dict: validators mapping, use (check, comparator) as key.
Examples:
>>> validators = [
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": {"b": 1}, "expect": 200, "comparator": "eq"}
]
>>> _convert_validators_to_mapping(validators)
{
("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
}
"""
validators_mapping = {}
for validator in validators:
if not isinstance(validator["check"], collections.Hashable):
check = json.dumps(validator["check"])
else:
check = validator["check"]
key = (check, validator["comparator"])
validators_mapping[key] = validator
return validators_mapping
def extend_validators(raw_validators, override_validators):
""" extend raw_validators with override_validators.
override_validators will merge and override raw_validators.
Args:
raw_validators (dict):
override_validators (dict):
Returns:
list: extended validators
Examples:
>>> raw_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}]
>>> override_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}]
>>> extend_validators(raw_validators, override_validators)
[
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": "s2", "expect": 16, "comparator": "len_eq"},
{"check": "s3", "expect": 12, "comparator": "len_eq"}
]
"""
if not raw_validators:
return override_validators
elif not override_validators:
return raw_validators
else:
def_validators_mapping = _convert_validators_to_mapping(raw_validators)
ref_validators_mapping = _convert_validators_to_mapping(override_validators)
def_validators_mapping.update(ref_validators_mapping)
return list(def_validators_mapping.values())
###############################################################################
## validate varibles and functions
###############################################################################
@@ -157,3 +326,22 @@ def is_variable(tup):
return False
return True
def validate_json_file(file_list):
""" validate JSON testcase format
"""
for json_file in set(file_list):
if not json_file.endswith(".json"):
logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file))
continue
logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN")
with io.open(json_file) as stream:
try:
json.load(stream)
except ValueError as e:
raise SystemExit(e)
print("OK")

2
tests/api/get_token.yml
View File

@@ -15,7 +15,7 @@ request:
Content-Type: "application/json"
device_sn: $device_sn
json:
sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)}
sign: ${get_sign($device_sn, $os_platform, $app_version)}
validate:
- eq: ["status_code", 0]
- len_eq: ["content.token", 12]

4
tests/api_server.py
View File

@@ -93,15 +93,13 @@ def index():
@app.route('/api/get-token', methods=['POST'])
def get_token():
user_agent = request.headers.get('User-Agent', "")
device_sn = request.headers.get('device_sn', "")
os_platform = request.headers.get('os_platform', "")
app_version = request.headers.get('app_version', "")
data = request.get_json()
sign = data.get('sign', "")
expected_sign = get_sign(user_agent, device_sn, os_platform, app_version)
expected_sign = get_sign(device_sn, os_platform, app_version)
if expected_sign != sign:
result = {
'success': False,

2
tests/base.py
View File

@@ -50,7 +50,7 @@ class ApiServerUnittest(unittest.TestCase):
'app_version': app_version
}
data = {
'sign': get_sign(user_agent, device_sn, os_platform, app_version)
'sign': get_sign(device_sn, os_platform, app_version)
}
resp = self.api_client.post(url, json=data, headers=headers)

13
tests/data/bugfix_verify.yml Normal file
View File

@@ -0,0 +1,13 @@
- config:
name: basic test with httpbin
base_url: https://httpbin.org/
verify: False
- test:
name: headers
request:
url: /headers
method: GET
validate:
- eq: ["status_code", 200]
- eq: [content.headers.Host, "httpbin.org"]

7
tests/data/demo_testcase.yml
View File

@@ -1,8 +1,9 @@
- config:
name: "123$var_a"
name: "123t$var_a"
variables:
var_a: 0
var_c: "${sum_two(1, 2)}"
var_a: 1
var_b: 2
var_c: "${sum_two($var_a, $var_b)}"
var_d: "${gen_random_string(5)}"
var_e: $var_d
PROJECT_KEY: ${ENV(PROJECT_KEY)}

2
tests/data/demo_testcase_cli.yml
View File

@@ -10,7 +10,7 @@
os_platform: 'ios'
app_version: '2.8.6'
json:
sign: f1219719911caae89ccc301679857ebfda115ca2
sign: 5188962c489d1a35effa99e9346dd5efd4fdabad
variables:
expect_status_code: 200
token_len: 16

2
tests/data/demo_testcase_functions.yml
View File

@@ -18,7 +18,7 @@
os_platform: $os_platform
app_version: $app_version
json:
sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)}
sign: ${get_sign($device_sn, $os_platform, $app_version)}
extract:
- token: content.token
validate:

2
tests/data/demo_testcase_hardcode.json
View File

@@ -13,7 +13,7 @@
"app_version": "2.8.6"
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"variables": [

2
tests/data/demo_testcase_hardcode.yml
View File

@@ -10,7 +10,7 @@
os_platform: 'ios'
app_version: '2.8.6'
json:
sign: f1219719911caae89ccc301679857ebfda115ca2
sign: 5188962c489d1a35effa99e9346dd5efd4fdabad
variables:
expect_status_code: 200
token_len: 16

2
tests/data/demo_testcase_variables.yml
View File

@@ -10,7 +10,7 @@
user_agent: 'iOS/10.3'
os_platform: 'ios'
app_version: '2.8.6'
sign: f1219719911caae89ccc301679857ebfda115ca2
sign: 5188962c489d1a35effa99e9346dd5efd4fdabad
request:
url: /api/get-token
method: POST

1
tests/httpbin/api/302_redirect.yml
View File

@@ -6,5 +6,6 @@ request:
url: https://debugtalk.com
status_code: 302
method: GET
verify: False
validate:
- eq: ["status_code", 200]

4
tests/locust_tests/demo_locusts.yml
View File

@@ -6,13 +6,13 @@ config:
testcases:
create user 1000 and check result.:
testcase: testcases/create_and_check.yml
testcase: testcases/create_user.yml
weight: 2
variables:
uid: 1000
create user 1001 and check result.:
testcase: testcases/create_and_check.yml
testcase: testcases/create_user.yml
weight: 3
variables:
uid: 1001

105
tests/test_api.py
View File

@@ -36,7 +36,7 @@ class TestHttpRunner(ApiServerUnittest):
'url': 'http://127.0.0.1:5000/api/get-token',
'method': 'POST',
'headers': {'Content-Type': 'application/json', 'app_version': '2.8.6', 'device_sn': 'FwgRiO7CNA50DSU', 'os_platform': 'ios', 'user_agent': 'iOS/10.3'},
'json': {'sign': '958a05393efef0ac7c0fb80a7eac45e24fd40c27'}
'json': {'sign': '9c0c7e51c91ae963c833a4ccbab8d683c4a90c98'}
},
'extract': [
{'token': 'content.token'}
@@ -52,7 +52,8 @@ class TestHttpRunner(ApiServerUnittest):
'request': {
'url': 'http://127.0.0.1:5000/api/users/1000',
'method': 'POST',
'headers': {'Content-Type': 'application/json', 'device_sn': 'FwgRiO7CNA50DSU','token': '$token'}, 'json': {'name': 'user1', 'password': '123456'}
'headers': {'Content-Type': 'application/json', 'device_sn': 'FwgRiO7CNA50DSU','token': '$token'},
'json': {'name': 'user1', 'password': '123456'}
},
'validate': [
{'eq': ['status_code', 201]},
@@ -137,11 +138,11 @@ class TestHttpRunner(ApiServerUnittest):
self.assertEqual(len(vars_out), 6)
self.assertEqual(vars_out[0]["in"]["uid"], 101)
self.assertEqual(vars_out[0]["in"]["device_sn"], "TESTSUITE_X1")
token1 = vars_out[0]["out"]["token"]
token1 = vars_out[0]["out"]["session_token"]
self.assertEqual(len(token1), 16)
self.assertEqual(vars_out[5]["in"]["uid"], 103)
self.assertEqual(vars_out[5]["in"]["device_sn"], "TESTSUITE_X2")
token2 = vars_out[0]["out"]["token"]
token2 = vars_out[0]["out"]["session_token"]
self.assertEqual(len(token2), 16)
self.assertEqual(token1, token2)
@@ -240,7 +241,7 @@ class TestHttpRunner(ApiServerUnittest):
summary = self.runner.summary
self.assertTrue(summary["success"])
self.assertEqual(summary["stat"]["testcases"]["total"], 2)
self.assertEqual(summary["stat"]["teststeps"]["total"], 8)
self.assertEqual(summary["stat"]["teststeps"]["total"], 4)
def test_run_httprunner_with_hooks(self):
testcase_file_path = os.path.join(
@@ -470,7 +471,7 @@ class TestHttpRunner(ApiServerUnittest):
self.assertEqual(len(summary["details"]), 3 * 2)
self.assertEqual(summary["stat"]["testcases"]["total"], 6)
self.assertEqual(summary["stat"]["teststeps"]["total"], 3 * 2 * 4)
self.assertEqual(summary["stat"]["teststeps"]["total"], 3 * 2 * 2)
self.assertEqual(
summary["details"][0]["name"],
"create user 101 and check result for TESTSUITE_X1."
@@ -481,10 +482,10 @@ class TestHttpRunner(ApiServerUnittest):
)
self.assertEqual(
summary["details"][0]["stat"]["total"],
4
2
)
records_name_list = [
summary["details"][i]["records"][2]["name"]
summary["details"][i]["records"][1]["meta_datas"][1]["name"]
for i in range(6)
]
self.assertEqual(
@@ -577,8 +578,7 @@ class TestApi(ApiServerUnittest):
testcase_path = "tests/testcases/setup.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
parsed_testcases = parsed_tests_mapping["testcases"]
parsed_testcases = parser.parse_tests(tests_mapping)
self.assertEqual(len(parsed_testcases), 1)
@@ -589,7 +589,7 @@ class TestApi(ApiServerUnittest):
self.assertEqual(test_dict1["name"], "get token (setup)")
self.assertNotIn("api_def", test_dict1)
self.assertEqual(test_dict1["variables"]["device_sn"], "TESTCASE_SETUP_XXX")
self.assertEqual(test_dict1["request"]["url"], "http://127.0.0.1:5000/api/get-token")
self.assertEqual(test_dict1["request"]["url"], "/api/get-token")
self.assertEqual(test_dict1["request"]["verify"], False)
test_dict2 = parsed_testcases[0]["teststeps"][1]
@@ -599,9 +599,9 @@ class TestApi(ApiServerUnittest):
testcase_path = "tests/testcases/setup.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
testcases = parser.parse_tests(tests_mapping)
runner = HttpRunner()
test_suite = runner._add_tests(parsed_tests_mapping)
test_suite = runner._add_tests(testcases)
self.assertEqual(len(test_suite._tests), 1)
teststeps = test_suite._tests[0].teststeps
@@ -610,38 +610,39 @@ class TestApi(ApiServerUnittest):
self.assertIn("api", teststeps[0])
def test_testcase_complex_verify(self):
testcase_path = "tests/testcases/create_and_check.yml"
testcase_path = "tests/testcases/create_user.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
teststeps = parsed_tests_mapping["testcases"][0]["teststeps"]
testcases = parser.parse_tests(tests_mapping)
teststeps = testcases[0]["teststeps"]
# testcases/setup.yml
teststep1 = teststeps[0]
self.assertEqual(teststep1["teststeps"][0]["request"]["verify"], False)
self.assertEqual(teststep1["teststeps"][1]["request"]["verify"], False)
teststep0 = teststeps[0]
self.assertEqual(teststep0["teststeps"][0]["request"]["verify"], False)
self.assertEqual(teststep0["teststeps"][1]["request"]["verify"], False)
# testcases/create_and_check.yml teststep 2/3/4
self.assertEqual(teststeps[1]["request"]["verify"], True)
self.assertEqual(teststeps[2]["request"]["verify"], True)
self.assertEqual(teststeps[3]["request"]["verify"], True)
# testcases/create_user.yml
teststep1 = teststeps[1]
self.assertEqual(teststep1["teststeps"][0]["request"]["verify"], True)
self.assertEqual(teststep1["teststeps"][1]["request"]["verify"], True)
self.assertEqual(teststep1["teststeps"][2]["request"]["verify"], True)
def test_testcase_simple_run_suite(self):
testcase_path = "tests/testcases/setup.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
testcases = parser.parse_tests(tests_mapping)
runner = HttpRunner()
test_suite = runner._add_tests(parsed_tests_mapping)
test_suite = runner._add_tests(testcases)
tests_results = runner._run_suite(test_suite)
self.assertEqual(len(tests_results[0][1].records), 2)
def test_testcase_complex_run_suite(self):
testcase_path = "tests/testcases/create_and_check.yml"
testcase_path = "tests/testcases/create_user.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
testcases = parser.parse_tests(tests_mapping)
runner = HttpRunner()
test_suite = runner._add_tests(parsed_tests_mapping)
test_suite = runner._add_tests(testcases)
tests_results = runner._run_suite(test_suite)
self.assertEqual(len(tests_results[0][1].records), 4)
self.assertEqual(len(tests_results[0][1].records), 2)
results = tests_results[0][1]
self.assertEqual(
@@ -650,7 +651,7 @@ class TestApi(ApiServerUnittest):
)
self.assertEqual(
results.records[1]["name"],
"make sure user 9001 does not exist"
"create user and check result."
)
def test_testsuite_loader(self):
@@ -679,7 +680,7 @@ class TestApi(ApiServerUnittest):
self.assertEqual(testcase_tests["name"], "create user 1000 and check result.")
self.assertIsInstance(testcase_tests["testcase_def"], dict)
self.assertEqual(testcase_tests["testcase_def"]["config"]["name"], "create user and check result.")
self.assertEqual(len(testcase_tests["testcase_def"]["teststeps"]), 4)
self.assertEqual(len(testcase_tests["testcase_def"]["teststeps"]), 2)
self.assertEqual(
testcase_tests["testcase_def"]["teststeps"][0]["name"],
"setup and reset all (override) for $device_sn."
@@ -689,57 +690,52 @@ class TestApi(ApiServerUnittest):
testcase_path = "tests/testsuites/create_users.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
parsed_testcases = parsed_tests_mapping["testcases"]
parsed_testcases = parser.parse_tests(tests_mapping)
self.assertEqual(len(parsed_testcases), 2)
self.assertEqual(len(parsed_testcases[0]["teststeps"]), 4)
self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2)
testcase1 = parsed_testcases[0]["teststeps"][0]
self.assertIn("setup and reset all (override)", testcase1["config"]["name"])
self.assertEqual(testcase1["teststeps"][0]["variables"]["var_c"], testcase1["teststeps"][0]["variables"]["var_d"])
self.assertEqual(testcase1["teststeps"][0]["variables"]["var_a"], testcase1["teststeps"][0]["variables"]["var_b"])
self.assertNotEqual(testcase1["teststeps"][0]["variables"]["var_a"], testcase1["teststeps"][0]["variables"]["var_c"])
self.assertIn("setup and reset all (override)", testcase1["config"]["name"].raw_string)
teststeps = testcase1["teststeps"]
self.assertNotIn("testcase_def", testcase1)
self.assertEqual(len(testcase1["teststeps"]), 2)
self.assertEqual(len(teststeps), 2)
self.assertEqual(
testcase1["teststeps"][0]["request"]["url"],
"http://127.0.0.1:5000/api/get-token"
teststeps[0]["request"]["url"],
"/api/get-token"
)
self.assertEqual(len(testcase1["teststeps"][0]["variables"]["device_sn"]), 15)
def test_testsuite_add_tests(self):
testcase_path = "tests/testsuites/create_users.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
testcases = parser.parse_tests(tests_mapping)
runner = HttpRunner()
test_suite = runner._add_tests(parsed_tests_mapping)
test_suite = runner._add_tests(testcases)
self.assertEqual(len(test_suite._tests), 2)
tests = test_suite._tests[0].teststeps
self.assertIn("setup and reset all (override)", tests[0]["config"]["name"])
self.assertIn("setup and reset all (override)", tests[0]["config"]["name"].raw_string)
def test_testsuite_run_suite(self):
testcase_path = "tests/testsuites/create_users.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
testcases = parser.parse_tests(tests_mapping)
runner = HttpRunner()
test_suite = runner._add_tests(parsed_tests_mapping)
test_suite = runner._add_tests(testcases)
tests_results = runner._run_suite(test_suite)
self.assertEqual(len(tests_results[0][1].records), 4)
self.assertEqual(len(tests_results[0][1].records), 2)
results = tests_results[0][1]
self.assertIn(
"setup and reset all (override)",
results.records[0]["name"]
)
self.assertIn(
self.assertEqual(
results.records[1]["name"],
["make sure user 1000 does not exist", "make sure user 1001 does not exist"]
"create user and check result."
)
@@ -749,11 +745,10 @@ class TestLocust(unittest.TestCase):
path = os.path.join(
os.getcwd(), 'tests/locust_tests/demo_locusts.yml')
locust_tests = prepare_locust_tests(path)
self.assertIn("gen_md5", locust_tests["functions"])
self.assertEqual(len(locust_tests["tests"]), 2 + 3)
self.assertEqual(len(locust_tests), 2 + 3)
name_list = [
"create user 1000 and check result.",
"create user 1001 and check result."
]
self.assertIn(locust_tests["tests"][0]["config"]["name"], name_list)
self.assertIn(locust_tests["tests"][4]["config"]["name"], name_list)
self.assertIn(locust_tests[0]["config"]["name"], name_list)
self.assertIn(locust_tests[4]["config"]["name"], name_list)

10
tests/test_client.py
View File

@@ -7,7 +7,7 @@ from tests.base import ApiServerUnittest
class TestHttpClient(ApiServerUnittest):
def setUp(self):
super(TestHttpClient, self).setUp()
self.api_client = HttpSession(self.host)
self.api_client = HttpSession()
self.headers = self.get_authenticated_headers()
self.reset_all()
@@ -30,7 +30,7 @@ class TestHttpClient(ApiServerUnittest):
self.assertEqual(True, resp.json()['success'])
def test_request_without_base_url(self):
url = "/api/users/1000"
url = "{}/api/users/1000".format(self.host)
data = {
'name': 'user1',
'password': '123456'
@@ -40,7 +40,7 @@ class TestHttpClient(ApiServerUnittest):
self.assertEqual(True, resp.json()['success'])
def test_request_post_data(self):
url = "/api/users/1000"
url = "{}/api/users/1000".format(self.host)
data = {
'name': 'user1',
'password': '123456'
@@ -56,7 +56,7 @@ class TestHttpClient(ApiServerUnittest):
self.assertIn("password=123456", resp.request.body)
def test_request_with_cookies(self):
url = "/api/users/1000"
url = "{}/api/users/1000".format(self.host)
data = {
'name': 'user1',
'password': '123456'
@@ -76,7 +76,7 @@ class TestHttpClient(ApiServerUnittest):
"a": "1",
"b": "2"
}
resp = self.api_client.get(url, cookies=cookies, headers=self.headers)
resp = self.api_client.get(url, cookies=cookies, headers=self.headers, verify=False)
raw_request = resp.history[0].request
self.assertEqual(raw_request._cookies["a"], "1")
self.assertEqual(raw_request._cookies["b"], "2")

182
tests/test_context.py
View File

@@ -1,9 +1,8 @@
import os
import time
import requests
from httprunner import context, exceptions, loader, response, utils
from tests.base import ApiServerUnittest
from httprunner import context, exceptions, loader, parser, runner
from tests.base import ApiServerUnittest, gen_md5, gen_random_string
class TestContext(ApiServerUnittest):
@@ -12,14 +11,9 @@ class TestContext(ApiServerUnittest):
loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
project_mapping = loader.project_mapping
self.context = context.SessionContext(
functions=project_mapping["functions"],
variables={"SECRET_KEY": "DebugTalk"}
)
def test_init_context_functions(self):
context_functions = self.context.FUNCTIONS_MAPPING
self.assertIn("gen_md5", context_functions)
def test_init_test_variables_initialize(self):
self.assertEqual(
self.context.test_variables_mapping,
@@ -30,16 +24,24 @@ class TestContext(ApiServerUnittest):
variables = {
"random": "${gen_random_string($num)}",
"authorization": "${gen_md5($TOKEN, $data, $random)}",
"data": '{"name": "$username", "password": "123456"}',
"data": "$username",
# TODO: escape '{' and '}'
# "data": '{"name": "$username", "password": "123456"}',
"TOKEN": "debugtalk",
"username": "user1",
"num": 6
}
functions = {
"gen_random_string": gen_random_string,
"gen_md5": gen_md5
}
variables = parser.prepare_lazy_data(variables, functions, variables.keys())
variables = parser.parse_variables_mapping(variables)
self.context.init_test_variables(variables)
variables_mapping = self.context.test_variables_mapping
self.assertEqual(len(variables_mapping["random"]), 6)
self.assertEqual(len(variables_mapping["authorization"]), 32)
self.assertEqual(variables_mapping["data"], '{"name": "user1", "password": "123456"}')
self.assertEqual(variables_mapping["data"], 'user1')
def test_update_seesion_variables(self):
self.context.update_session_variables({"TOKEN": "debugtalk"})
@@ -48,15 +50,11 @@ class TestContext(ApiServerUnittest):
"debugtalk"
)
def test_eval_content_functions(self):
content = "${sleep_N_secs(1)}"
start_time = time.time()
self.context.eval_content(content)
elapsed_time = time.time() - start_time
self.assertGreater(elapsed_time, 1)
def test_eval_content_variables(self):
content = "abc$SECRET_KEY"
variables = {
"SECRET_KEY": "DebugTalk"
}
content = parser.prepare_lazy_data("abc$SECRET_KEY", {}, variables.keys())
self.assertEqual(
self.context.eval_content(content),
"abcDebugTalk"
@@ -76,7 +74,12 @@ class TestContext(ApiServerUnittest):
"authorization": "${gen_md5($TOKEN, $data, $random)}",
"TOKEN": "debugtalk"
}
functions = {
"gen_random_string": gen_random_string,
"gen_md5": gen_md5
}
variables = parser.prepare_lazy_data(variables, functions, variables.keys())
variables = parser.parse_variables_mapping(variables)
self.context.init_test_variables(variables)
request = {
@@ -90,7 +93,12 @@ class TestContext(ApiServerUnittest):
},
"data": "$data"
}
parsed_request = self.context.eval_content(request)
prepared_request = parser.prepare_lazy_data(
request,
functions,
{"authorization", "random", "SECRET_KEY", "data"}
)
parsed_request = self.context.eval_content(prepared_request)
self.assertIn("authorization", parsed_request["headers"])
self.assertEqual(len(parsed_request["headers"]["authorization"]), 32)
self.assertIn("random", parsed_request["headers"])
@@ -102,74 +110,80 @@ class TestContext(ApiServerUnittest):
)
self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk")
def test_do_validation(self):
self.context._do_validation(
{"check": "check", "check_value": 1, "expect": 1, "comparator": "eq"}
)
self.context._do_validation(
{"check": "check", "check_value": "abc", "expect": "abc", "comparator": "=="}
)
self.context._do_validation(
{"check": "status_code", "check_value": "201", "expect": 3, "comparator": "sum_status_code"}
)
def test_validate(self):
url = "http://127.0.0.1:5000/"
resp = requests.get(url)
resp_obj = response.ResponseObject(resp)
validators = [
{"eq": ["$resp_status_code", 201]},
{"check": "$resp_status_code", "comparator": "eq", "expect": 201},
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
testcases = [
{
"config": {
'name': "test validation"
},
"teststeps": [
{
"name": "test validation",
"request": {
"url": "http://127.0.0.1:5000/",
"method": "GET",
},
"variables": {
"resp_status_code": 200,
"resp_body_success": True
},
"validate": [
{"eq": ["$resp_status_code", 200]},
{"check": "$resp_status_code", "comparator": "eq", "expect": 200},
{"check": "$resp_body_success", "expect": True},
{"check": "${is_status_code_200($resp_status_code)}", "expect": True}
]
}
]
}
]
variables = {
"resp_status_code": 200,
"resp_body_success": True
from tests.debugtalk import is_status_code_200
tests_mapping = {
"project_mapping": {
"functions": {
"is_status_code_200": is_status_code_200
}
},
"testcases": testcases
}
self.context.init_test_variables(variables)
with self.assertRaises(exceptions.ValidationFailure):
self.context.validate(validators, resp_obj)
validators = [
{"eq": ["$resp_status_code", 201]},
{"check": "$resp_status_code", "comparator": "eq", "expect": 201},
{"check": "$resp_body_success", "comparator": "eq", "expect": True},
{"check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False}
]
variables = [
{"resp_status_code": 201},
{"resp_body_success": True}
]
self.context.init_test_variables(variables)
self.context.validate(validators, resp_obj)
self.context.validate([], resp_obj)
self.assertEqual(self.context.validation_results, [])
testcases = parser.parse_tests(tests_mapping)
parsed_testcase = testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
teststep = parsed_testcase["teststeps"][0]
test_runner.run_test(teststep)
def test_validate_exception(self):
url = "http://127.0.0.1:5000/"
resp = requests.get(url)
resp_obj = response.ResponseObject(resp)
# expected value missed in validators
validators = [
{"eq": ["$resp_status_code", 201]},
{"check": "$resp_status_code", "comparator": "eq", "expect": 201}
testcases = [
{
"config": {
'name': "test validation"
},
"teststeps": [
{
"name": "test validation",
"request": {
"url": "http://127.0.0.1:5000/",
"method": "GET",
},
"variables": {
"resp_status_code": 200,
"resp_body_success": True
},
"validate": [
{"eq": ["$resp_status_code", 201]},
{"check": "$resp_status_code", "expect": 201},
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
]
}
]
}
]
variables = []
self.context.init_test_variables(variables)
with self.assertRaises(exceptions.VariableNotFound):
self.context.validate(validators, resp_obj)
# expected value missed in variables mapping
variables = [
{"resp_status_code": 200}
]
self.context.init_test_variables(variables)
tests_mapping = {
"testcases": testcases
}
testcases = parser.parse_tests(tests_mapping)
parsed_testcase = testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
teststep = parsed_testcase["teststeps"][0]
with self.assertRaises(exceptions.ValidationFailure):
self.context.validate(validators, resp_obj)
test_runner.run_test(teststep)

4
tests/test_loader.py
View File

@@ -348,14 +348,14 @@ class TestSuiteLoader(unittest.TestCase):
tests_mapping = loader.load_tests(testcase_file_path)
testcases = tests_mapping["testcases"]
self.assertIsInstance(testcases, list)
self.assertEqual(testcases[0]["config"]["name"], '123$var_a')
self.assertEqual(testcases[0]["config"]["name"], '123t$var_a')
self.assertIn(
"sum_two",
tests_mapping["project_mapping"]["functions"]
)
self.assertEqual(
testcases[0]["config"]["variables"]["var_c"],
"${sum_two(1, 2)}"
"${sum_two($var_a, $var_b)}"
)
self.assertEqual(
testcases[0]["config"]["variables"]["PROJECT_KEY"],

587
tests/test_parser.py
View File

@@ -1,11 +1,13 @@
import os
import re
import time
import unittest
from httprunner import exceptions, loader, parser
from tests.debugtalk import gen_random_string, sum_two
class TestParser(unittest.TestCase):
class TestParserBasic(unittest.TestCase):
def test_parse_string_value(self):
self.assertEqual(parser.parse_string_value("123"), 123)
@@ -14,139 +16,170 @@ class TestParser(unittest.TestCase):
self.assertEqual(parser.parse_string_value("$var"), "$var")
self.assertEqual(parser.parse_string_value("${func}"), "${func}")
def test_extract_variables(self):
def test_regex_findall_variables(self):
self.assertEqual(
parser.extract_variables("$var"),
parser.regex_findall_variables("$var"),
["var"]
)
self.assertEqual(
parser.extract_variables("$var123"),
parser.regex_findall_variables("$var123"),
["var123"]
)
self.assertEqual(
parser.extract_variables("$var_name"),
parser.regex_findall_variables("$var_name"),
["var_name"]
)
self.assertEqual(
parser.extract_variables("var"),
parser.regex_findall_variables("var"),
[]
)
self.assertEqual(
parser.extract_variables("a$var"),
parser.regex_findall_variables("a$var"),
["var"]
)
self.assertEqual(
parser.extract_variables("$v ar"),
parser.regex_findall_variables("$v ar"),
["v"]
)
self.assertEqual(
parser.extract_variables(" "),
parser.regex_findall_variables(" "),
[]
)
self.assertEqual(
parser.extract_variables("$abc*"),
parser.regex_findall_variables("$abc*"),
["abc"]
)
self.assertEqual(
parser.extract_variables("${func()}"),
parser.regex_findall_variables("${func()}"),
[]
)
self.assertEqual(
parser.extract_variables("${func(1,2)}"),
parser.regex_findall_variables("${func(1,2)}"),
[]
)
self.assertEqual(
parser.extract_variables("${gen_md5($TOKEN, $data, $random)}"),
parser.regex_findall_variables("${gen_md5($TOKEN, $data, $random)}"),
["TOKEN", "data", "random"]
)
def test_parse_function(self):
def test_parse_function_params(self):
self.assertEqual(
parser.parse_function("func()"),
{'func_name': 'func', 'args': [], 'kwargs': {}}
parser.parse_function_params(""),
{'args': [], 'kwargs': {}}
)
self.assertEqual(
parser.parse_function("func(5)"),
{'func_name': 'func', 'args': [5], 'kwargs': {}}
parser.parse_function_params("5"),
{'args': [5], 'kwargs': {}}
)
self.assertEqual(
parser.parse_function("func(1, 2)"),
{'func_name': 'func', 'args': [1, 2], 'kwargs': {}}
parser.parse_function_params("1, 2"),
{'args': [1, 2], 'kwargs': {}}
)
self.assertEqual(
parser.parse_function("func(a=1, b=2)"),
{'func_name': 'func', 'args': [], 'kwargs': {'a': 1, 'b': 2}}
parser.parse_function_params("a=1, b=2"),
{'args': [], 'kwargs': {'a': 1, 'b': 2}}
)
self.assertEqual(
parser.parse_function("func(a= 1, b =2)"),
{'func_name': 'func', 'args': [], 'kwargs': {'a': 1, 'b': 2}}
parser.parse_function_params("a= 1, b =2"),
{'args': [], 'kwargs': {'a': 1, 'b': 2}}
)
self.assertEqual(
parser.parse_function("func(1, 2, a=3, b=4)"),
{'func_name': 'func', 'args': [1, 2], 'kwargs': {'a': 3, 'b': 4}}
parser.parse_function_params("1, 2, a=3, b=4"),
{'args': [1, 2], 'kwargs': {'a': 3, 'b': 4}}
)
self.assertEqual(
parser.parse_function("func($request, 123)"),
{'func_name': 'func', 'args': ["$request", 123], 'kwargs': {}}
parser.parse_function_params("$request, 123"),
{'args': ["$request", 123], 'kwargs': {}}
)
self.assertEqual(
parser.parse_function("func( )"),
{'func_name': 'func', 'args': [], 'kwargs': {}}
parser.parse_function_params(" "),
{'args': [], 'kwargs': {}}
)
self.assertEqual(
parser.parse_function("func(hello world, a=3, b=4)"),
{'func_name': 'func', 'args': ["hello world"], 'kwargs': {'a': 3, 'b': 4}}
parser.parse_function_params("hello world, a=3, b=4"),
{'args': ["hello world"], 'kwargs': {'a': 3, 'b': 4}}
)
self.assertEqual(
parser.parse_function("func($request, 12 3)"),
{'func_name': 'func', 'args': ["$request", '12 3'], 'kwargs': {}}
parser.parse_function_params("$request, 12 3"),
{'args': ["$request", '12 3'], 'kwargs': {}}
)
def test_parse_validator(self):
validator = {"check": "status_code", "comparator": "eq", "expect": 201}
def test_extract_variables(self):
prepared_content = parser.prepare_lazy_data("123$a", {}, {"a"})
self.assertEqual(
parser.parse_validator(validator),
{"check": "status_code", "comparator": "eq", "expect": 201}
parser.extract_variables(prepared_content),
{"a"}
)
validator = {'eq': ['status_code', 201]}
prepared_content = parser.prepare_lazy_data("$a$b", {}, {"a", "b"})
self.assertEqual(
parser.parse_validator(validator),
{"check": "status_code", "comparator": "eq", "expect": 201}
parser.extract_variables(prepared_content),
{"a", "b"}
)
prepared_content = parser.prepare_lazy_data(["$a$b", "$c", "d"], {}, {"a", "b", "c", "d"})
self.assertEqual(
parser.extract_variables(prepared_content),
{"a", "b", "c"}
)
prepared_content = parser.prepare_lazy_data(
{"a": 1, "b": {"c": "$d", "e": 3}},
{},
{"d"}
)
self.assertEqual(
parser.extract_variables(prepared_content),
{"d"}
)
prepared_content = parser.prepare_lazy_data(
{"a": ["$b"], "b": {"c": "$d", "e": 3}},
{},
{"b", "d"}
)
self.assertEqual(
parser.extract_variables(prepared_content),
{"b", "d"}
)
prepared_content = parser.prepare_lazy_data(
["$a$b", "$c", {"c": "$d"}],
{},
{"a", "b", "c", "d"}
)
self.assertEqual(
parser.extract_variables(prepared_content),
{"a", "b", "c", "d"}
)
def test_extract_functions(self):
self.assertEqual(
parser.extract_functions("${func()}"),
["func()"]
parser.regex_findall_functions("${func()}"),
[('func', '')]
)
self.assertEqual(
parser.extract_functions("${func(5)}"),
["func(5)"]
parser.regex_findall_functions("${func(5)}"),
[('func', '5')]
)
self.assertEqual(
parser.extract_functions("${func(a=1, b=2)}"),
["func(a=1, b=2)"]
parser.regex_findall_functions("${func(a=1, b=2)}"),
[('func', 'a=1, b=2')]
)
self.assertEqual(
parser.extract_functions("${func(1, $b, c=$x, d=4)}"),
["func(1, $b, c=$x, d=4)"]
parser.regex_findall_functions("${func(1, $b, c=$x, d=4)}"),
[('func', '1, $b, c=$x, d=4')]
)
self.assertEqual(
parser.extract_functions("/api/1000?_t=${get_timestamp()}"),
["get_timestamp()"]
parser.regex_findall_functions("/api/1000?_t=${get_timestamp()}"),
[('get_timestamp', '')]
)
self.assertEqual(
parser.extract_functions("/api/${add(1, 2)}"),
["add(1, 2)"]
parser.regex_findall_functions("/api/${add(1, 2)}"),
[('add', '1, 2')]
)
self.assertEqual(
parser.extract_functions("/api/${add(1, 2)}?_t=${get_timestamp()}"),
["add(1, 2)", "get_timestamp()"]
parser.regex_findall_functions("/api/${add(1, 2)}?_t=${get_timestamp()}"),
[('add', '1, 2'), ('get_timestamp', '')]
)
self.assertEqual(
parser.extract_functions("abc${func(1, 2, a=3, b=4)}def"),
["func(1, 2, a=3, b=4)"]
parser.regex_findall_functions("abc${func(1, 2, a=3, b=4)}def"),
[('func', '1, 2, a=3, b=4')]
)
def test_parse_data(self):
@@ -172,7 +205,7 @@ class TestParser(unittest.TestCase):
functions_mapping = {
"add_one": lambda x: x + 1
}
result = parser.parse_data(content, variables_mapping, functions_mapping)
result = parser.eval_lazy_data(content, variables_mapping, functions_mapping)
self.assertEqual("/api/users/1000", result["request"]["url"])
self.assertEqual("abc123", result["request"]["headers"]["token"])
self.assertEqual("POST", result["request"]["method"])
@@ -182,7 +215,7 @@ class TestParser(unittest.TestCase):
self.assertEqual("", result["request"]["data"]["empty_str"])
self.assertEqual("abc4def", result["request"]["data"]["value"])
def test_parse_data_variables(self):
def test_eval_lazy_data(self):
variables_mapping = {
"var_1": "abc",
"var_2": "def",
@@ -192,66 +225,150 @@ class TestParser(unittest.TestCase):
"var_6": None
}
self.assertEqual(
parser.parse_data("$var_1", variables_mapping),
parser.eval_lazy_data("$var_1", variables_mapping=variables_mapping),
"abc"
)
self.assertEqual(
parser.parse_data("var_1", variables_mapping),
parser.eval_lazy_data("var_1", variables_mapping=variables_mapping),
"var_1"
)
self.assertEqual(
parser.parse_data("$var_1#XYZ", variables_mapping),
parser.eval_lazy_data("$var_1#XYZ", variables_mapping=variables_mapping),
"abc#XYZ"
)
self.assertEqual(
parser.parse_data("/$var_1/$var_2/var3", variables_mapping),
parser.eval_lazy_data("/$var_1/$var_2/var3", variables_mapping=variables_mapping),
"/abc/def/var3"
)
self.assertEqual(
parser.parse_data("/$var_1/$var_2/$var_1", variables_mapping),
parser.eval_lazy_data("/$var_1/$var_2/$var_1", variables_mapping=variables_mapping),
"/abc/def/abc"
)
self.assertEqual(
parser.parse_string_variables("${func($var_1, $var_2, xyz)}", variables_mapping, {}),
"${func(abc, def, xyz)}"
)
self.assertEqual(
parser.parse_data("$var_3", variables_mapping),
parser.eval_lazy_data("$var_3", variables_mapping=variables_mapping),
123
)
self.assertEqual(
parser.parse_data("$var_4", variables_mapping),
parser.eval_lazy_data("$var_4", variables_mapping=variables_mapping),
{"a": 1}
)
self.assertEqual(
parser.parse_data("$var_5", variables_mapping),
parser.eval_lazy_data("$var_5", variables_mapping=variables_mapping),
True
)
self.assertEqual(
parser.parse_data("abc$var_5", variables_mapping),
parser.eval_lazy_data("abc$var_5", variables_mapping=variables_mapping),
"abcTrue"
)
self.assertEqual(
parser.parse_data("abc$var_4", variables_mapping),
parser.eval_lazy_data("abc$var_4", variables_mapping=variables_mapping),
"abc{'a': 1}"
)
self.assertEqual(
parser.parse_data("$var_6", variables_mapping),
parser.eval_lazy_data("$var_6", variables_mapping=variables_mapping),
None
)
with self.assertRaises(exceptions.VariableNotFound):
parser.parse_data("/api/$SECRET_KEY", variables_mapping)
parser.eval_lazy_data("/api/$SECRET_KEY", variables_mapping=variables_mapping)
self.assertEqual(
parser.parse_data(["$var_1", "$var_2"], variables_mapping),
parser.eval_lazy_data(["$var_1", "$var_2"], variables_mapping=variables_mapping),
["abc", "def"]
)
self.assertEqual(
parser.parse_data({"$var_1": "$var_2"}, variables_mapping),
parser.eval_lazy_data({"$var_1": "$var_2"}, variables_mapping=variables_mapping),
{"abc": "def"}
)
def test_lazy_string(self):
variables_mapping = {
"var_1": "abc",
"var_2": "def",
"var_3": 123,
"var_4": {"a": 1},
"var_5": True,
"var_6": None
}
check_variables_set = variables_mapping.keys()
functions_mapping = {
"func1": lambda x,y: str(x) + str(y)
}
var = parser.LazyString("ABC$var_1", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}")
self.assertEqual(var._args, ["var_1"])
self.assertEqual(var.to_value(variables_mapping), "ABCabc")
var = parser.LazyString("ABC$var_1$var_3", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}{}")
self.assertEqual(var._args, ["var_1", "var_3"])
self.assertEqual(var.to_value(variables_mapping), "ABCabc123")
var = parser.LazyString("ABC$var_1/$var_3", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}/{}")
self.assertEqual(var._args, ["var_1", "var_3"])
self.assertEqual(var.to_value(variables_mapping), "ABCabc/123")
var = parser.LazyString("ABC$var_1/", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}/")
self.assertEqual(var._args, ["var_1"])
self.assertEqual(var.to_value(variables_mapping), "ABCabc/")
var = parser.LazyString("ABC$var_1$", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}$")
self.assertEqual(var._args, ["var_1"])
self.assertEqual(var.to_value(variables_mapping), "ABCabc$")
var = parser.LazyString("ABC$var_1{", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}{")
self.assertEqual(var._args, ["var_1"])
# self.assertEqual(var.to_value(variables_mapping), "ABCabc{")
var = parser.LazyString("ABC$$var_1{", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC${}{")
self.assertEqual(var._args, ["var_1"])
var = parser.LazyString("ABC$var_1${", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}${")
self.assertEqual(var._args, ["var_1"])
var = parser.LazyString("ABC$var_1${a", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}${a")
self.assertEqual(var._args, ["var_1"])
var = parser.LazyString("ABC$var_1/$var_2/$var_1", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}/{}/{}")
self.assertEqual(var._args, ["var_1", "var_2", "var_1"])
self.assertEqual(var.to_value(variables_mapping), "ABCabc/def/abc")
var = parser.LazyString("func1($var_1, $var_3)", functions_mapping, check_variables_set)
self.assertEqual(var._string, "func1({}, {})")
self.assertEqual(var._args, ["var_1", "var_3"])
self.assertEqual(var.to_value(variables_mapping), "func1(abc, 123)")
var = parser.LazyString("${func1($var_1, $var_3)}", functions_mapping, check_variables_set)
self.assertEqual(var._string, "{}")
self.assertIsInstance(var._args[0], parser.LazyFunction)
self.assertEqual(var.to_value(variables_mapping), "abc123")
var = parser.LazyString("ABC${func1($var_1, $var_3)}DE", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}DE")
self.assertIsInstance(var._args[0], parser.LazyFunction)
self.assertEqual(var.to_value(variables_mapping), "ABCabc123DE")
var = parser.LazyString("ABC${func1($var_1, $var_3)}$var_5", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}{}")
self.assertEqual(var.to_value(variables_mapping), "ABCabc123True")
var = parser.LazyString("ABC${func1($var_1, $var_3)}DE$var_4", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}DE{}")
self.assertEqual(var.to_value(variables_mapping), "ABCabc123DE{'a': 1}")
var = parser.LazyString("ABC$var_5${func1($var_1, $var_3)}", functions_mapping, check_variables_set)
self.assertEqual(var._string, "ABC{}{}")
self.assertEqual(var.to_value(variables_mapping), "ABCTrueabc123")
def test_parse_data_multiple_identical_variables(self):
variables_mapping = {
"userid": 100,
@@ -259,7 +376,7 @@ class TestParser(unittest.TestCase):
}
content = "/users/$userid/training/$data?userId=$userid&data=$data"
self.assertEqual(
parser.parse_data(content, variables_mapping),
parser.eval_lazy_data(content, variables_mapping=variables_mapping),
"/users/100/training/1498?userId=100&data=1498"
)
@@ -270,36 +387,35 @@ class TestParser(unittest.TestCase):
}
content = "/users/$user/$userid/$data?userId=$userid&data=$data"
self.assertEqual(
parser.parse_data(content, variables_mapping),
parser.eval_lazy_data(content, variables_mapping=variables_mapping),
"/users/100/1000/1498?userId=1000&data=1498"
)
def test_parse_data_functions(self):
import random, string
functions_mapping = {
"gen_random_string": lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) \
for _ in range(str_len))
"gen_random_string": gen_random_string
}
result = parser.parse_data("${gen_random_string(5)}", functions_mapping=functions_mapping)
result = parser.eval_lazy_data("${gen_random_string(5)}", functions_mapping=functions_mapping)
self.assertEqual(len(result), 5)
add_two_nums = lambda a, b=1: a + b
functions_mapping["add_two_nums"] = add_two_nums
self.assertEqual(
parser.parse_data("${add_two_nums(1)}", functions_mapping=functions_mapping),
parser.eval_lazy_data("${add_two_nums(1)}", functions_mapping=functions_mapping),
2
)
self.assertEqual(
parser.parse_data("${add_two_nums(1, 2)}", functions_mapping=functions_mapping),
parser.eval_lazy_data("${add_two_nums(1, 2)}", functions_mapping=functions_mapping),
3
)
self.assertEqual(
parser.parse_data("/api/${add_two_nums(1, 2)}", functions_mapping=functions_mapping),
parser.eval_lazy_data("/api/${add_two_nums(1, 2)}", functions_mapping=functions_mapping),
"/api/3"
)
with self.assertRaises(exceptions.FunctionNotFound):
parser.parse_data("/api/${gen_md5(abc)}")
parser.eval_lazy_data("/api/${gen_md5(abc)}", functions_mapping=functions_mapping)
def test_parse_data_testcase(self):
variables = {
@@ -323,7 +439,11 @@ class TestParser(unittest.TestCase):
},
"body": "$data"
}
parsed_testcase = parser.parse_data(testcase_template, variables, functions)
parsed_testcase = parser.eval_lazy_data(
testcase_template,
variables_mapping=variables,
functions_mapping=functions
)
self.assertEqual(
parsed_testcase["url"],
"http://127.0.0.1:5000/api/users/1000/3"
@@ -345,25 +465,133 @@ class TestParser(unittest.TestCase):
3
)
def test_substitute_variables(self):
content = {
'request': {
'url': '/api/users/$uid?id=$id',
'headers': {'token': '$token'}
}
def test_parse_variables_mapping(self):
variables = {
"varA": "123$varB",
"varB": "456$varC",
"varC": "${sum_two($a, $b)}",
"a": 1,
"b": 2
}
variables_mapping = {"$uid": 1000, "$id": 2}
substituted_data = parser.substitute_variables(content, variables_mapping)
self.assertEqual(substituted_data["request"]["url"], "/api/users/1000?id=2")
self.assertEqual(substituted_data["request"]["headers"], {'token': '$token'})
functions = {
"sum_two": sum_two
}
prepared_variables = parser.prepare_lazy_data(variables, functions, variables.keys())
parsed_variables = parser.parse_variables_mapping(prepared_variables)
self.assertEqual(parsed_variables["varA"], "1234563")
self.assertEqual(parsed_variables["varB"], "4563")
self.assertEqual(parsed_variables["varC"], 3)
def test_parse_variables_mapping_fix_duplicate_function_call(self):
# fix duplicate function calling
variables = {
"varA": "$varB",
"varB": "${gen_random_string(5)}"
}
functions = {
"gen_random_string": gen_random_string
}
prepared_variables = parser.prepare_lazy_data(variables, functions, variables.keys())
parsed_variables = parser.parse_variables_mapping(prepared_variables)
self.assertEqual(parsed_variables["varA"], parsed_variables["varB"])
def test_parse_variables_mapping_dead_circle(self):
variables = {
"varA": "$varB",
"varB": "123$varC"
}
check_variables_set = {"varA", "varB", "varC"}
prepared_variables = parser.prepare_lazy_data(variables, {}, check_variables_set)
with self.assertRaises(exceptions.VariableNotFound):
parser.parse_variables_mapping(prepared_variables)
def test_parse_variables_mapping_not_found(self):
variables = {
"varA": "123$varB",
"varB": "456$varC",
"varC": "${sum_two($a, $b)}",
"b": 2
}
functions = {
"sum_two": sum_two
}
with self.assertRaises(exceptions.VariableNotFound):
parser.prepare_lazy_data(variables, functions, variables.keys())
def test_parse_variables_mapping_ref_self(self):
variables = {
"varC": "${sum_two($a, $b)}",
"a": 1,
"b": 2,
"token": "$token"
}
functions = {
"sum_two": sum_two
}
prepared_variables = parser.prepare_lazy_data(variables, functions, variables.keys())
with self.assertRaises(exceptions.VariableNotFound):
parser.parse_variables_mapping(prepared_variables)
def test_parse_variables_mapping_2(self):
variables = {
"host2": "https://httprunner.org",
"num3": "${sum_two($num2, 4)}",
"num2": "${sum_two($num1, 3)}",
"num1": "${sum_two(1, 2)}"
}
functions = {
"sum_two": sum_two
}
prepared_variables = parser.prepare_lazy_data(variables, functions, variables.keys())
parsed_testcase = parser.parse_variables_mapping(prepared_variables)
self.assertEqual(parsed_testcase["num3"], 10)
self.assertEqual(parsed_testcase["num2"], 6)
self.assertEqual(parsed_testcase["num1"], 3)
def test_prepare_lazy_data(self):
variables = {
"host": "https://httprunner.org",
"num4": "${sum_two($num0, 5)}",
"num3": "${sum_two($num2, 4)}",
"num2": "${sum_two($num1, 3)}",
"num1": "${sum_two(1, 2)}",
"num0": 0
}
functions = {
"sum_two": sum_two
}
parser.prepare_lazy_data(
variables,
functions,
variables.keys()
)
def test_prepare_lazy_data_not_found(self):
variables = {
"host": "https://httprunner.org",
"num4": "${sum_two($num0, 5)}",
"num3": "${sum_two($num2, 4)}",
"num2": "${sum_two($num1, 3)}",
"num1": "${sum_two(1, 2)}"
}
functions = {
"sum_two": sum_two
}
with self.assertRaises(exceptions.VariableNotFound):
parser.prepare_lazy_data(
variables,
functions,
variables.keys()
)
class TestParser(unittest.TestCase):
def test_parse_parameters_raw_list(self):
parameters = [
{"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]},
{"username-password": [("user1", "111111"), ["test2", "222222"]]}
]
variables_mapping = {}
functions_mapping = {}
cartesian_product_parameters = parser.parse_parameters(parameters)
self.assertEqual(
len(cartesian_product_parameters),
@@ -441,20 +669,18 @@ class TestParser(unittest.TestCase):
testcases = tests_mapping["testcases"]
self.assertEqual(
testcases[0]["config"]["variables"]["var_c"],
"${sum_two(1, 2)}"
"${sum_two($var_a, $var_b)}"
)
self.assertEqual(
testcases[0]["config"]["variables"]["PROJECT_KEY"],
"${ENV(PROJECT_KEY)}"
)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
parsed_testcases = parsed_tests_mapping["testcases"]
parsed_testcases = parser.parse_tests(tests_mapping)
self.assertIsInstance(parsed_testcases, list)
test_dict1 = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict1["variables"]["var_c"], 3)
self.assertEqual(test_dict1["variables"]["PROJECT_KEY"], "ABCDEFGH")
self.assertEqual(test_dict1["variables"]["var_d"], test_dict1["variables"]["var_e"])
self.assertEqual(parsed_testcases[0]["config"]["name"], '1230')
self.assertEqual(test_dict1["variables"]["var_c"].raw_string, "${sum_two($var_a, $var_b)}")
self.assertEqual(test_dict1["variables"]["PROJECT_KEY"].raw_string, "${ENV(PROJECT_KEY)}")
self.assertIsInstance(parsed_testcases[0]["config"]["name"], parser.LazyString)
def test_parse_tests_override_variables(self):
tests_mapping = {
@@ -480,10 +706,10 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict1_variables = parsed_tests_mapping["testcases"][0]["teststeps"][0]["variables"]
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict1_variables = parsed_testcases[0]["teststeps"][0]["variables"]
self.assertEqual(test_dict1_variables["creator"], "user_test_001")
self.assertEqual(test_dict1_variables["username"], "user_test_001")
self.assertEqual(test_dict1_variables["username"].raw_string, "$creator")
def test_parse_tests_base_url_priority(self):
""" base_url & verify: priority test_dict > config
@@ -509,9 +735,9 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0]
self.assertEqual(test_dict["request"]["url"], "https://httprunner.org/api1")
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict["request"]["url"], "/api1")
self.assertEqual(test_dict["request"]["verify"], True)
def test_parse_tests_base_url_path_with_variable(self):
@@ -537,9 +763,11 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0]
self.assertEqual(test_dict["request"]["url"], "https://httprunner.org/api1")
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict["variables"]["host2"], "https://httprunner.org")
parsed_test_dict = parser.parse_lazy_data(test_dict, test_dict["variables"])
self.assertEqual(parsed_test_dict["request"]["url"], "https://httprunner.org/api1")
def test_parse_tests_base_url_test_dict(self):
tests_mapping = {
@@ -565,54 +793,12 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0]
self.assertEqual(test_dict["request"]["url"], "https://httprunner.org/api1")
def test_parse_data_with_variables(self):
variables = {
"host2": "https://httprunner.org",
"num3": "${sum_two($num2, 4)}",
"num2": "${sum_two($num1, 3)}",
"num1": "${sum_two(1, 2)}"
}
from tests.debugtalk import sum_two
functions = {
"sum_two": sum_two
}
parsed_testcase = parser.parse_data(variables, variables, functions)
self.assertEqual(parsed_testcase["num3"], 10)
self.assertEqual(parsed_testcase["num2"], 6)
self.assertEqual(parsed_testcase["num1"], 3)
def test_parse_data_with_variables_not_found(self):
variables = {
"host": "https://httprunner.org",
"num4": "${sum_two($num0, 5)}",
"num3": "${sum_two($num2, 4)}",
"num2": "${sum_two($num1, 3)}",
"num1": "${sum_two(1, 2)}"
}
from tests.debugtalk import sum_two
functions = {
"sum_two": sum_two
}
with self.assertRaises(exceptions.VariableNotFound):
parser.parse_data(variables, variables, functions)
parsed_testcase = parser.parse_data(
variables,
variables,
functions,
raise_if_variable_not_found=False
)
self.assertEqual(parsed_testcase["num3"], 10)
self.assertEqual(parsed_testcase["num2"], 6)
self.assertEqual(parsed_testcase["num1"], 3)
self.assertEqual(parsed_testcase["num4"], "${sum_two($num0, 5)}")
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
parsed_test_dict = parser.parse_lazy_data(test_dict, test_dict["variables"])
self.assertEqual(parsed_test_dict["base_url"], "https://httprunner.org")
def test_parse_tests_variable_with_function(self):
from tests.debugtalk import sum_two, gen_random_string
tests_mapping = {
"project_mapping": {
"functions": {
@@ -652,18 +838,20 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0]
self.assertEqual(test_dict["variables"]["num3"], 10)
self.assertEqual(test_dict["variables"]["num2"], 6)
self.assertEqual(test_dict["variables"]["str1"], test_dict["variables"]["str2"])
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
variables = parser.parse_variables_mapping(test_dict["variables"])
self.assertEqual(variables["num3"], 10)
self.assertEqual(variables["num2"], 6)
parsed_test_dict = parser.parse_lazy_data(test_dict, variables)
self.assertEqual(parsed_test_dict["base_url"], "https://httprunner.org")
self.assertEqual(
test_dict["request"]["url"],
"https://httprunner.org/api1/?num1=3&num2=6&num3=10"
parsed_test_dict["request"]["url"],
"/api1/?num1=3&num2=6&num3=10"
)
self.assertEqual(variables["str1"], variables["str2"])
def test_parse_tests_variable_not_found(self):
from tests.debugtalk import sum_two
tests_mapping = {
"project_mapping": {
"functions": {
@@ -699,15 +887,8 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0]
self.assertEqual(test_dict["variables"]["num3"], 10)
self.assertEqual(test_dict["variables"]["num2"], 6)
self.assertEqual(test_dict["variables"]["num4"], "${sum_two($num0, 5)}")
self.assertEqual(
test_dict["request"]["url"],
"https://httprunner.org/api1/?num1=3&num2=6&num3=10&num4=${sum_two($num0, 5)}"
)
with self.assertRaises(exceptions.VariableNotFound):
parser.parse_tests(tests_mapping)
def test_parse_tests_base_url_teststep_empty(self):
""" base_url & verify: priority test_dict > config
@@ -733,9 +914,9 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0]
self.assertEqual(test_dict["request"]["url"], "https://debugtalk.com/api1")
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
self.assertEqual(str(test_dict["base_url"]), 'LazyString($host)')
self.assertEqual(test_dict["request"]["verify"], True)
def test_parse_tests_verify_config_set(self):
@@ -758,8 +939,8 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0]
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict["request"]["verify"], False)
def test_parse_tests_verify_config_unset(self):
@@ -781,8 +962,8 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0]
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict["request"]["verify"], True)
def test_parse_tests_verify_step_set_false(self):
@@ -805,8 +986,8 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0]
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict["request"]["verify"], False)
def test_parse_tests_verify_nested_testcase_unset(self):
@@ -840,8 +1021,8 @@ class TestParser(unittest.TestCase):
}
]
}
parsed_tests_mapping = parser.parse_tests(tests_mapping)
test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0]
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict["teststeps"][0]["request"]["verify"], False)
def test_parse_environ(self):
@@ -851,7 +1032,7 @@ class TestParser(unittest.TestCase):
{"PROJECT_KEY": "${ENV(PROJECT_KEY)}"}
]
}
result = parser.parse_data(content)
result = parser.eval_lazy_data(content)
content = {
"variables": [
@@ -859,7 +1040,7 @@ class TestParser(unittest.TestCase):
]
}
with self.assertRaises(exceptions.ParamsError):
parser.parse_data(content)
parser.eval_lazy_data(content)
content = {
"variables": [
@@ -867,7 +1048,7 @@ class TestParser(unittest.TestCase):
]
}
with self.assertRaises(exceptions.ParamsError):
parser.parse_data(content)
parser.eval_lazy_data(content)
def test_extend_with_api(self):
loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
@@ -888,18 +1069,18 @@ class TestParser(unittest.TestCase):
'url': '/api/get-token',
'method': 'POST',
'headers': {'user_agent': '$user_agent', 'device_sn': '$device_sn', 'os_platform': '$os_platform', 'app_version': '$app_version'},
'json': {'sign': '${get_sign($user_agent, $device_sn, $os_platform, $app_version)}'}
'json': {'sign': '${get_sign($device_sn, $os_platform, $app_version)}'}
},
'validate': [
{'eq': ['status_code', 201]},
{'len_eq': ['content.token', 32]}
{"check": "status_code", "comparator": "equals", "expect": 201},
{"check": "content.token", "comparator": "length_equals", "expect": 32}
]
}
extended_block = parser._extend_with_api(test_block, api_def_dict)
self.assertEqual(extended_block["base_url"], "https://debugtalk.com")
self.assertEqual(extended_block["name"], "override block")
self.assertEqual({'var': 123}, extended_block["variables"])
self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, extended_block["validate"])
self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, extended_block["validate"])
self.assertEqual(extended_block["times"], 3)
parser._extend_with_api(test_block, api_def_dict)
self.assertEqual(test_block["base_url"], "https://debugtalk.com")
self.assertEqual(test_block["name"], "override block")
self.assertEqual({'var': 123}, test_block["variables"])
self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'equals'}, test_block["validate"])
self.assertIn({'check': 'content.token', 'comparator': 'length_equals', 'expect': 32}, test_block["validate"])
self.assertEqual(test_block["times"], 3)

435
tests/test_runner.py
View File

@@ -1,8 +1,7 @@
import os
import time
from httprunner import exceptions, loader, runner
from httprunner.utils import deep_update_dict
from httprunner import loader, parser, runner
from tests.api_server import HTTPBIN_SERVER
from tests.base import ApiServerUnittest
@@ -19,7 +18,7 @@ class TestRunner(ApiServerUnittest):
"base_url": "http://127.0.0.1",
"verify": False
}
self.test_runner = runner.Runner(config, self.debugtalk_functions)
self.test_runner = runner.Runner(config)
self.reset_all()
def reset_all(self):
@@ -36,204 +35,253 @@ class TestRunner(ApiServerUnittest):
]
for testcase_file_path in testcase_file_path_list:
testcases = loader.load_file(testcase_file_path)
config_dict = {}
test_runner = runner.Runner(config_dict, self.debugtalk_functions)
test = testcases[0]["test"]
test_runner.run_test(test)
test = testcases[1]["test"]
test_runner.run_test(test)
test = testcases[2]["test"]
test_runner.run_test(test)
def test_run_single_testcase_fail(self):
test = {
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
},
"validate": [
{"check": "status_code", "expect": 205},
{"check": "content.token", "comparator": "len_eq", "expect": 19}
]
}
with self.assertRaises(exceptions.ValidationFailure):
self.test_runner.run_test(test)
tests_mapping = loader.load_tests(testcase_file_path)
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
test_runner.run_test(parsed_testcase["teststeps"][1])
test_runner.run_test(parsed_testcase["teststeps"][2])
def test_run_testcase_with_hooks(self):
start_time = time.time()
config_dict = {
"name": "basic test with httpbin",
"base_url": HTTPBIN_SERVER,
"setup_hooks": [
"${sleep_N_secs(0.5)}"
"${hook_print(setup)}"
],
"teardown_hooks": [
"${sleep_N_secs(1)}",
"${hook_print(teardown)}"
]
}
test = {
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
testcases = [
{
"config": {
"name": "basic test with httpbin",
"base_url": HTTPBIN_SERVER,
"setup_hooks": [
"${sleep_N_secs(0.5)}",
"${hook_print(setup)}"
],
"teardown_hooks": [
"${sleep_N_secs(1)}",
"${hook_print(teardown)}"
]
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
"teststeps": [
{
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 200}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"validate": [
{"check": "status_code", "expect": 200}
]
"testcases": testcases
}
test_runner = runner.Runner(config_dict, self.debugtalk_functions)
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
end_time = time.time()
# check if testcase setup hook executed
self.assertGreater(end_time - start_time, 0.5)
start_time = time.time()
test_runner.run_test(test)
test_runner.run_test(test)
test_runner.run_test(parsed_testcase["teststeps"][0])
end_time = time.time()
# testcase teardown hook has not been executed now
self.assertLess(end_time - start_time, 1)
def test_run_testcase_with_hooks_assignment(self):
config_dict = {
"name": "basic test with httpbin",
"base_url": HTTPBIN_SERVER
}
test = {
"name": "modify request headers",
"request": {
"url": "/anything",
"method": "POST",
"headers": {
"user_agent": "iOS/10.3",
"os_platform": "ios"
testcases = [
{
"config": {
"name": "basic test with httpbin",
"base_url": HTTPBIN_SERVER
},
"data": "a=1&b=2"
"teststeps": [
{
"name": "modify request headers",
"base_url": HTTPBIN_SERVER,
"request": {
"url": "/anything",
"method": "POST",
"headers": {
"user_agent": "iOS/10.3",
"os_platform": "ios"
},
"data": "a=1&b=2"
},
"setup_hooks": [
{"total": "${sum_two(1, 5)}"}
],
"validate": [
{"check": "status_code", "expect": 200}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"setup_hooks": [
{"total": "${sum_two(1, 5)}"}
],
"validate": [
{"check": "status_code", "expect": 200}
]
"testcases": testcases
}
test_runner = runner.Runner(config_dict, self.debugtalk_functions)
test_runner.run_test(test)
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
test_variables_mapping = test_runner.session_context.test_variables_mapping
self.assertEqual(test_variables_mapping["total"], 6)
self.assertEqual(test_variables_mapping["request"]["data"], "a=1&b=2")
def test_run_testcase_with_hooks_modify_request(self):
config_dict = {
"name": "basic test with httpbin",
"base_url": HTTPBIN_SERVER
}
test = {
"name": "modify request headers",
"request": {
"url": "/anything",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3"
testcases = [
{
"config": {
"name": "basic test with httpbin",
"base_url": HTTPBIN_SERVER
},
"json": {
"os_platform": "ios",
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
"teststeps": [
{
"name": "modify request headers",
"base_url": HTTPBIN_SERVER,
"request": {
"url": "/anything",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3"
},
"json": {
"os_platform": "ios",
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"setup_hooks": [
"${modify_request_json($request, android)}"
],
"validate": [
{"check": "status_code", "expect": 200},
{"check": "content.json.os_platform", "expect": "android"}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"setup_hooks": [
"${modify_request_json($request, android)}"
],
"validate": [
{"check": "status_code", "expect": 200},
{"check": "content.json.os_platform", "expect": "android"}
]
"testcases": testcases
}
test_runner = runner.Runner(config_dict, self.debugtalk_functions)
test_runner.run_test(test)
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
def test_run_testcase_with_teardown_hooks_success(self):
test = {
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
testcases = [
{
"config": {
"name": "basic test with httpbin"
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
"teststeps": [
{
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 200}
],
"teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"validate": [
{"check": "status_code", "expect": 200}
],
"teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"]
"testcases": testcases
}
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
start_time = time.time()
self.test_runner.run_test(test)
test_runner.run_test(parsed_testcase["teststeps"][0])
end_time = time.time()
# check if teardown function executed
self.assertLess(end_time - start_time, 0.5)
def test_run_testcase_with_teardown_hooks_fail(self):
test = {
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token2",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
testcases = [
{
"config": {
"name": "basic test with httpbin"
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
"teststeps": [
{
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token2",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 404}
],
"teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"validate": [
{"check": "status_code", "expect": 404}
],
"teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"]
"testcases": testcases
}
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
start_time = time.time()
self.test_runner.run_test(test)
test_runner.run_test(parsed_testcase["teststeps"][0])
end_time = time.time()
# check if teardown function executed
self.assertGreater(end_time - start_time, 2)
@@ -241,34 +289,51 @@ class TestRunner(ApiServerUnittest):
def test_bugfix_type_match(self):
testcase_file_path = os.path.join(
os.getcwd(), 'tests/data/bugfix_type_match.yml')
testcases = loader.load_file(testcase_file_path)
test = testcases[1]["test"]
self.test_runner.run_test(test)
tests_mapping = loader.load_tests(testcase_file_path)
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
def test_run_validate_elapsed(self):
test = {
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
testcases = [
{
"config": {},
"teststeps": [
{
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 200},
{"check": "elapsed.seconds", "comparator": "lt", "expect": 1},
{"check": "elapsed.days", "comparator": "eq", "expect": 0},
{"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000},
{"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"validate": [
{"check": "status_code", "expect": 200},
{"check": "elapsed.seconds", "comparator": "lt", "expect": 1},
{"check": "elapsed.days", "comparator": "eq", "expect": 0},
{"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000},
{"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1}
]
"testcases": testcases
}
self.test_runner.run_test(test)
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])

84
tests/test_utils.py
View File

@@ -61,35 +61,6 @@ class TestUtils(ApiServerUnittest):
result = utils.query_json(json_content, query)
self.assertEqual(result, "L")
def test_get_uniform_comparator(self):
self.assertEqual(utils.get_uniform_comparator("eq"), "equals")
self.assertEqual(utils.get_uniform_comparator("=="), "equals")
self.assertEqual(utils.get_uniform_comparator("lt"), "less_than")
self.assertEqual(utils.get_uniform_comparator("le"), "less_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("gt"), "greater_than")
self.assertEqual(utils.get_uniform_comparator("ge"), "greater_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("ne"), "not_equals")
self.assertEqual(utils.get_uniform_comparator("str_eq"), "string_equals")
self.assertEqual(utils.get_uniform_comparator("len_eq"), "length_equals")
self.assertEqual(utils.get_uniform_comparator("count_eq"), "length_equals")
self.assertEqual(utils.get_uniform_comparator("len_gt"), "length_greater_than")
self.assertEqual(utils.get_uniform_comparator("count_gt"), "length_greater_than")
self.assertEqual(utils.get_uniform_comparator("count_greater_than"), "length_greater_than")
self.assertEqual(utils.get_uniform_comparator("len_ge"), "length_greater_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("count_ge"), "length_greater_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("count_greater_than_or_equals"), "length_greater_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("len_lt"), "length_less_than")
self.assertEqual(utils.get_uniform_comparator("count_lt"), "length_less_than")
self.assertEqual(utils.get_uniform_comparator("count_less_than"), "length_less_than")
self.assertEqual(utils.get_uniform_comparator("len_le"), "length_less_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("count_le"), "length_less_than_or_equals")
self.assertEqual(utils.get_uniform_comparator("count_less_than_or_equals"), "length_less_than_or_equals")
def current_validators(self):
from httprunner import built_in
functions_mapping = loader.load_module_functions(built_in)
@@ -205,61 +176,6 @@ class TestUtils(ApiServerUnittest):
self.assertIsInstance(ordered_dict, dict)
self.assertIn("a", ordered_dict)
def test_extend_validators(self):
def_validators = [
{'eq': ['v1', 200]},
{"check": "s2", "expect": 16, "comparator": "len_eq"}
]
current_validators = [
{"check": "v1", "expect": 201},
{'len_eq': ['s3', 12]}
]
def_validators = [
parser.parse_validator(validator)
for validator in def_validators
]
ref_validators = [
parser.parse_validator(validator)
for validator in current_validators
]
extended_validators = utils.extend_validators(def_validators, ref_validators)
self.assertIn(
{"check": "v1", "expect": 201, "comparator": "eq"},
extended_validators
)
self.assertIn(
{"check": "s2", "expect": 16, "comparator": "len_eq"},
extended_validators
)
self.assertIn(
{"check": "s3", "expect": 12, "comparator": "len_eq"},
extended_validators
)
def test_extend_validators_with_dict(self):
def_validators = [
{'eq': ["a", {"v": 1}]},
{'eq': [{"b": 1}, 200]}
]
current_validators = [
{'len_eq': ['s3', 12]},
{'eq': [{"b": 1}, 201]}
]
def_validators = [
parser.parse_validator(validator)
for validator in def_validators
]
ref_validators = [
parser.parse_validator(validator)
for validator in current_validators
]
extended_validators = utils.extend_validators(def_validators, ref_validators)
self.assertEqual(len(extended_validators), 3)
self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'eq'}, extended_validators)
self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'eq'}, extended_validators)
def test_extend_variables(self):
raw_variables = [{"var1": "val1"}, {"var2": "val2"}]
override_variables = [{"var1": "val111"}, {"var3": "val3"}]

98
tests/test_validator.py
View File

@@ -74,3 +74,101 @@ class TestValidator(unittest.TestCase):
func = lambda x: x + 1
self.assertTrue(validator.is_function(func))
self.assertTrue(validator.is_function(validator.is_testcase))
def test_get_uniform_comparator(self):
self.assertEqual(validator.get_uniform_comparator("eq"), "equals")
self.assertEqual(validator.get_uniform_comparator("=="), "equals")
self.assertEqual(validator.get_uniform_comparator("lt"), "less_than")
self.assertEqual(validator.get_uniform_comparator("le"), "less_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("gt"), "greater_than")
self.assertEqual(validator.get_uniform_comparator("ge"), "greater_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("ne"), "not_equals")
self.assertEqual(validator.get_uniform_comparator("str_eq"), "string_equals")
self.assertEqual(validator.get_uniform_comparator("len_eq"), "length_equals")
self.assertEqual(validator.get_uniform_comparator("count_eq"), "length_equals")
self.assertEqual(validator.get_uniform_comparator("len_gt"), "length_greater_than")
self.assertEqual(validator.get_uniform_comparator("count_gt"), "length_greater_than")
self.assertEqual(validator.get_uniform_comparator("count_greater_than"), "length_greater_than")
self.assertEqual(validator.get_uniform_comparator("len_ge"), "length_greater_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("count_ge"), "length_greater_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("count_greater_than_or_equals"), "length_greater_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("len_lt"), "length_less_than")
self.assertEqual(validator.get_uniform_comparator("count_lt"), "length_less_than")
self.assertEqual(validator.get_uniform_comparator("count_less_than"), "length_less_than")
self.assertEqual(validator.get_uniform_comparator("len_le"), "length_less_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("count_le"), "length_less_than_or_equals")
self.assertEqual(validator.get_uniform_comparator("count_less_than_or_equals"), "length_less_than_or_equals")
def test_parse_validator(self):
_validator = {"check": "status_code", "comparator": "eq", "expect": 201}
self.assertEqual(
validator.uniform_validator(_validator),
{"check": "status_code", "comparator": "equals", "expect": 201}
)
_validator = {'eq': ['status_code', 201]}
self.assertEqual(
validator.uniform_validator(_validator),
{"check": "status_code", "comparator": "equals", "expect": 201}
)
def test_extend_validators(self):
def_validators = [
{'eq': ['v1', 200]},
{"check": "s2", "expect": 16, "comparator": "len_eq"}
]
current_validators = [
{"check": "v1", "expect": 201},
{'len_eq': ['s3', 12]}
]
def_validators = [
validator.uniform_validator(_validator)
for _validator in def_validators
]
ref_validators = [
validator.uniform_validator(_validator)
for _validator in current_validators
]
extended_validators = validator.extend_validators(def_validators, ref_validators)
self.assertIn(
{"check": "v1", "expect": 201, "comparator": "equals"},
extended_validators
)
self.assertIn(
{"check": "s2", "expect": 16, "comparator": "length_equals"},
extended_validators
)
self.assertIn(
{"check": "s3", "expect": 12, "comparator": "length_equals"},
extended_validators
)
def test_extend_validators_with_dict(self):
def_validators = [
{'eq': ["a", {"v": 1}]},
{'eq': [{"b": 1}, 200]}
]
current_validators = [
{'len_eq': ['s3', 12]},
{'eq': [{"b": 1}, 201]}
]
def_validators = [
validator.uniform_validator(_validator)
for _validator in def_validators
]
ref_validators = [
validator.uniform_validator(_validator)
for _validator in current_validators
]
extended_validators = validator.extend_validators(def_validators, ref_validators)
self.assertEqual(len(extended_validators), 3)
self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'equals'}, extended_validators)
self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'equals'}, extended_validators)

22
tests/testcases/create_user.yml Normal file
View File

@@ -0,0 +1,22 @@
- config:
name: "create user and check result."
id: create_user
base_url: "http://127.0.0.1:5000"
variables:
uid: 9001
device_sn: "TESTCASE_CREATE_XXX"
output:
- session_token
- test:
name: setup and reset all (override) for $device_sn.
testcase: testcases/setup.yml
output:
- session_token
- test:
name: create user and check result.
variables:
token: $session_token
testcase: testcases/deps/check_and_create.yml

9
tests/testcases/create_and_check.yml → tests/testcases/deps/check_and_create.yml
View File

@@ -1,4 +1,3 @@
- config:
name: "create user and check result."
id: create_and_check
@@ -6,14 +5,6 @@
variables:
uid: 9001
device_sn: "TESTCASE_CREATE_XXX"
output:
- token
- test:
name: setup and reset all (override) for $device_sn.
testcase: testcases/setup.yml
output:
- token
- test:
name: make sure user $uid does not exist

6
tests/testcases/setup.yml
View File

@@ -9,7 +9,7 @@
base_url: "http://127.0.0.1:5000"
verify: False
output:
- token
- session_token
- test:
name: get token (setup)
@@ -20,7 +20,7 @@
os_platform: 'ios'
app_version: '2.8.6'
extract:
- token: content.token
- session_token: content.token
validate:
- eq: ["status_code", 200]
- len_eq: ["content.token", 16]
@@ -29,4 +29,4 @@
name: reset all users
api: api/reset_all.yml
variables:
token: $token
token: $session_token

4
tests/testsuites/create_users.yml
View File

@@ -8,14 +8,14 @@ config:
testcases:
create user 1000 and check result.:
testcase: testcases/create_and_check.yml
testcase: testcases/create_user.yml
variables:
uid: 1000
var_c: ${gen_random_string(5)}
var_d: $var_c
create user 1001 and check result.:
testcase: testcases/create_and_check.yml
testcase: testcases/create_user.yml
variables:
uid: 1001
var_c: ${gen_random_string(5)}

2
tests/testsuites/create_users_with_parameters.yml
View File

@@ -6,7 +6,7 @@ config:
testcases:
create user $uid and check result for $device_sn.:
testcase: testcases/create_and_check.yml
testcase: testcases/create_user.yml
variables:
uid: 1000
device_sn: TESTSUITE_XXX