GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-05-12 10:49:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1 from httprunner/dev

Browse Source 
Dev
This commit is contained in:
t-bug
2019-12-05 19:11:57 +08:00
committed by GitHub
parent 19254e9295 67725d5860
commit d54416d5d4
20 changed files with 672 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
docs/CHANGELOG.md
View File

@@ -1,5 +1,21 @@
# Release History
## 2.4.0 (2019-12-04)
**Added**
- feat: validate with python script, ref #773
## 2.3.3 (2019-12-04)
**Fixed**
- fix #768: dump json file path error when folder name contains dot, such as `a.b.c`
**Changed**
- change: rename builtin function, sleep_N_secs => sleep
## 2.3.2 (2019-11-01)
**Added**

2
httprunner/__init__.py
View File

@@ -1,4 +1,4 @@
__version__ = "2.3.2"
__version__ = "2.4.0"
__description__ = "One-stop solution for HTTP(S) testing."
__all__ = ["__version__", "__description__"]

5
httprunner/api.py
View File

@@ -258,7 +258,6 @@ class HttpRunner(object):
# load tests
self.exception_stage = "load tests"
tests_mapping = loader.load_tests(path, dot_env_path)
tests_mapping["project_mapping"]["test_path"] = path
if mapping:
tests_mapping["project_mapping"]["variables"] = mapping
@@ -278,9 +277,9 @@ class HttpRunner(object):
"""
logger.log_info("HttpRunner version: {}".format(__version__))
if validator.is_testcase_path(path_or_tests):
if loader.is_testcase_path(path_or_tests):
return self.run_path(path_or_tests, dot_env_path, mapping)
elif validator.is_testcases(path_or_tests):
elif loader.is_testcases(path_or_tests):
return self.run_tests(path_or_tests)
else:
raise exceptions.ParamsError("Invalid testcase path or testcases: {}".format(path_or_tests))

2
httprunner/cli.py
View File

@@ -5,11 +5,11 @@ import sys
from httprunner import __description__, __version__
from httprunner.api import HttpRunner
from httprunner.compat import is_py2
from httprunner.loader import validate_json_file
from httprunner.logger import color_print
from httprunner.report import gen_html_report
from httprunner.utils import (create_scaffold, get_python2_retire_msg,
prettify_json_file)
from httprunner.validator import validate_json_file
def main():

14
httprunner/client.py
View File

@@ -8,7 +8,7 @@ from requests import Request, Response
from requests.exceptions import (InvalidSchema, InvalidURL, MissingSchema,
RequestException)
from httprunner import logger
from httprunner import logger, response
from httprunner.utils import lower_dict_keys, omit_long_data
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -115,7 +115,10 @@ class HttpSession(requests.Session):
else:
try:
# try to record json data
req_resp_dict["response"]["json"] = resp_obj.json()
if isinstance(resp_obj, response.ResponseObject):
req_resp_dict["response"]["json"] = resp_obj.json
else:
req_resp_dict["response"]["json"] = resp_obj.json()
except ValueError:
# only record at most 512 text charactors
resp_text = resp_obj.text
@@ -126,6 +129,13 @@ class HttpSession(requests.Session):
return req_resp_dict
def update_last_req_resp_record(self, resp_obj):
"""
update request and response info from Response() object.
"""
self.meta_data["data"].pop()
self.meta_data["data"].append(self.get_req_resp_record(resp_obj))
def request(self, method, url, name=None, **kwargs):
"""
Constructs and sends a :py:class:`requests.Request`.

112
httprunner/context.py
View File

@@ -1,4 +1,4 @@
from httprunner import exceptions, logger, parser, utils
from httprunner import parser, utils
class SessionContext(object):
@@ -13,11 +13,12 @@ class SessionContext(object):
>>> context.update_session_variables(variables)
"""
def __init__(self, variables=None):
variables_mapping = utils.ensure_mapping_format(variables or {})
self.session_variables_mapping = parser.parse_variables_mapping(variables_mapping)
self.test_variables_mapping = {}
self.init_test_variables()
self.validation_results = []
def init_test_variables(self, variables_mapping=None):
""" init test variables, called when each test(api) starts.
@@ -61,110 +62,3 @@ class SessionContext(object):
content may be in any data structure, include dict, list, tuple, number, string, etc.
"""
return parser.parse_lazy_data(content, self.test_variables_mapping)
def __eval_validator_check(self, check_item, resp_obj):
""" evaluate check item in validator.
Args:
check_item: check_item should only be the following 5 formats:
1, variable reference, e.g. $token
2, function reference, e.g. ${is_status_code_200($status_code)}
3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
4, string joined by delimiter. e.g. "status_code", "headers.content-type"
5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
resp_obj: response object
"""
if isinstance(check_item, (dict, list)) \
or isinstance(check_item, parser.LazyString):
# format 1/2/3
check_value = self.eval_content(check_item)
else:
# format 4/5
check_value = resp_obj.extract_field(check_item)
return check_value
def __eval_validator_expect(self, expect_item):
""" evaluate expect item in validator.
Args:
expect_item: expect_item should only be in 2 types:
1, variable reference, e.g. $expect_status_code
2, actual value, e.g. 200
"""
expect_value = self.eval_content(expect_item)
return expect_value
def validate(self, validators, resp_obj):
""" make validation with comparators
"""
self.validation_results = []
if not validators:
return
logger.log_debug("start to validate.")
validate_pass = True
failures = []
for validator in validators:
# validator should be LazyFunction object
if not isinstance(validator, parser.LazyFunction):
raise exceptions.ValidationFailure(
"validator should be parsed first: {}".format(validators))
# evaluate validator args with context variable mapping.
validator_args = validator.get_args()
check_item, expect_item = validator_args
check_value = self.__eval_validator_check(
check_item,
resp_obj
)
expect_value = self.__eval_validator_expect(expect_item)
validator.update_args([check_value, expect_value])
comparator = validator.func_name
validator_dict = {
"comparator": comparator,
"check": check_item,
"check_value": check_value,
"expect": expect_item,
"expect_value": expect_value
}
validate_msg = "\nvalidate: {} {} {}({})".format(
check_item,
comparator,
expect_value,
type(expect_value).__name__
)
try:
validator.to_value(self.test_variables_mapping)
validator_dict["check_result"] = "pass"
validate_msg += "\t==> pass"
logger.log_debug(validate_msg)
except (AssertionError, TypeError):
validate_pass = False
validator_dict["check_result"] = "fail"
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__,
comparator,
expect_value,
type(expect_value).__name__
)
logger.log_error(validate_msg)
failures.append(validate_msg)
self.validation_results.append(validator_dict)
# restore validator args, in case of running multiple times
validator.update_args(validator_args)
if not validate_pass:
failures_string = "\n".join([failure for failure in failures])
raise exceptions.ValidationFailure(failures_string)

195
httprunner/loader.py
View File

@@ -4,6 +4,7 @@ import io
import json
import os
import sys
import types
import yaml
@@ -17,6 +18,149 @@ except AttributeError:
pass
# TODO: validate data format with JSON schema
def is_testcase(data_structure):
""" check if data_structure is a testcase.
Args:
data_structure (dict): testcase should always be in the following data structure:
{
"config": {
"name": "desc1",
"variables": [], # optional
"request": {} # optional
},
"teststeps": [
test_dict1,
{ # test_dict2
'name': 'test step desc2',
'variables': [], # optional
'extract': [], # optional
'validate': [],
'request': {},
'function_meta': {}
}
]
}
Returns:
bool: True if data_structure is valid testcase, otherwise False.
"""
# TODO: replace with JSON schema validation
if not isinstance(data_structure, dict):
return False
if "teststeps" not in data_structure:
return False
if not isinstance(data_structure["teststeps"], list):
return False
return True
def is_testcases(data_structure):
""" check if data_structure is testcase or testcases list.
Args:
data_structure (dict): testcase(s) should always be in the following data structure:
{
"project_mapping": {
"PWD": "XXXXX",
"functions": {},
"env": {}
},
"testcases": [
{ # testcase data structure
"config": {
"name": "desc1",
"path": "testcase1_path",
"variables": [], # optional
},
"teststeps": [
# test data structure
{
'name': 'test step desc1',
'variables': [], # optional
'extract': [], # optional
'validate': [],
'request': {}
},
test_dict_2 # another test dict
]
},
testcase_dict_2 # another testcase dict
]
}
Returns:
bool: True if data_structure is valid testcase(s), otherwise False.
"""
if not isinstance(data_structure, dict):
return False
if "testcases" not in data_structure:
return False
testcases = data_structure["testcases"]
if not isinstance(testcases, list):
return False
for item in testcases:
if not is_testcase(item):
return False
return True
def is_testcase_path(path):
""" check if path is testcase path or path list.
Args:
path (str/list): file path or file path list.
Returns:
bool: True if path is valid file path or path list, otherwise False.
"""
if not isinstance(path, (str, list)):
return False
if isinstance(path, list):
for p in path:
if not is_testcase_path(p):
return False
if isinstance(path, str):
if not os.path.exists(path):
return False
return True
def validate_json_file(file_list):
""" validate JSON testcase format
"""
for json_file in set(file_list):
if not json_file.endswith(".json"):
logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file))
continue
logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN")
with io.open(json_file) as stream:
try:
json.load(stream)
except ValueError as e:
raise SystemExit(e)
print("OK")
###############################################################################
# file loader
###############################################################################
@@ -244,6 +388,31 @@ def locate_file(start_path, file_name):
###############################################################################
def is_function(item):
""" Takes item object, returns True if it is a function.
"""
return isinstance(item, types.FunctionType)
def is_variable(tup):
""" Takes (name, object) tuple, returns True if it is a variable.
"""
name, item = tup
if callable(item):
# function or class
return False
if isinstance(item, types.ModuleType):
# imported module
return False
if name.startswith("_"):
# private property
return False
return True
def load_module_functions(module):
""" load python module functions.
@@ -262,7 +431,7 @@ def load_module_functions(module):
module_functions = {}
for name, item in vars(module).items():
if validator.is_function(item):
if is_function(item):
module_functions[name] = item
return module_functions
@@ -724,7 +893,7 @@ def load_api_folder(api_folder_path):
for api_item in api_items:
key, api_dict = api_item.popitem()
api_id = api_dict.get("id") or api_dict.get("def") \
or api_dict.get("name")
or api_dict.get("name")
if key != "api" or not api_id:
raise exceptions.ParamsError(
"Invalid API defined in {}".format(api_file_path))
@@ -779,6 +948,19 @@ def load_project_tests(test_path, dot_env_path=None):
environments and debugtalk.py functions.
"""
def prepare_path(path):
if not os.path.exists(path):
err_msg = "path not exist: {}".format(path)
logger.log_error(err_msg)
raise exceptions.FileNotFound(err_msg)
if not os.path.isabs(path):
path = os.path.join(os.getcwd(), path)
return path
test_path = prepare_path(test_path)
# locate debugtalk.py file
debugtalk_path = locate_debugtalk_py(test_path)
@@ -810,6 +992,7 @@ def load_project_tests(test_path, dot_env_path=None):
project_mapping["PWD"] = project_working_directory
built_in.PWD = project_working_directory
project_mapping["functions"] = debugtalk_functions
project_mapping["test_path"] = test_path
# load api
tests_def_mapping["api"] = load_api_folder(os.path.join(project_working_directory, "api"))
@@ -869,14 +1052,6 @@ def load_tests(path, dot_env_path=None):
}
"""
if not os.path.exists(path):
err_msg = "path not exist: {}".format(path)
logger.log_error(err_msg)
raise exceptions.FileNotFound(err_msg)
if not os.path.isabs(path):
path = os.path.join(os.getcwd(), path)
load_project_tests(path, dot_env_path)
tests_mapping = {
"project_mapping": project_mapping

40
httprunner/runner.py
View File

@@ -5,6 +5,7 @@ from unittest.case import SkipTest
from httprunner import exceptions, logger, response, utils
from httprunner.client import HttpSession
from httprunner.context import SessionContext
from httprunner.validator import Validator
class Runner(object):
@@ -62,7 +63,6 @@ class Runner(object):
"""
self.verify = config.get("verify", True)
self.export = config.get("export") or config.get("output", [])
self.validation_results = []
config_variables = config.get("variables", {})
# testcase setup hooks
@@ -86,19 +86,8 @@ class Runner(object):
if not isinstance(self.http_client_session, HttpSession):
return
self.validation_results = []
self.http_client_session.init_meta_data()
def __get_test_data(self):
""" get request/response data and validate results
"""
if not isinstance(self.http_client_session, HttpSession):
return
meta_data = self.http_client_session.meta_data
meta_data["validators"] = self.validation_results
return meta_data
def _handle_skip_feature(self, test_dict):
""" handle skip feature for test
- skip: skip current test unconditionally
@@ -244,7 +233,8 @@ class Runner(object):
raise exceptions.ParamsError(err_msg)
logger.log_info("{method} {url}".format(method=method, url=parsed_url))
logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
logger.log_debug(
"request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
# request
resp = self.http_client_session.request(
@@ -260,6 +250,7 @@ class Runner(object):
if teardown_hooks:
self.session_context.update_test_variables("response", resp_obj)
self.do_hook_actions(teardown_hooks, "teardown")
self.http_client_session.update_last_req_resp_record(resp_obj)
# extract
extractors = test_dict.get("extract", {})
@@ -268,9 +259,18 @@ class Runner(object):
# validate
validators = test_dict.get("validate") or test_dict.get("validators") or []
validate_script = test_dict.get("validate_script", [])
if validate_script:
validators.append({
"type": "python_script",
"script": validate_script
})
validator = Validator(self.session_context, resp_obj)
try:
self.session_context.validate(validators, resp_obj)
except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure):
validator.validate(validators)
except (exceptions.ParamsError,
exceptions.ValidationFailure, exceptions.ExtractFailure):
err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32)
# log request
@@ -294,7 +294,9 @@ class Runner(object):
raise
finally:
self.validation_results = self.session_context.validation_results
# get request/response data and validate results
self.meta_datas = getattr(self.http_client_session, "meta_data", {})
self.meta_datas["validators"] = validator.validation_results
def _run_testcase(self, testcase_dict):
""" run single testcase.
@@ -379,8 +381,6 @@ class Runner(object):
self.exception_request_type = test_dict["request"]["method"]
self.exception_name = test_dict.get("name")
raise
finally:
self.meta_datas = self.__get_test_data()
def export_variables(self, output_variables_list):
""" export current testcase variables
@@ -391,8 +391,8 @@ class Runner(object):
for variable in output_variables_list:
if variable not in variables_mapping:
logger.log_warning(
"variable '{}' can not be found in variables mapping, failed to export!"\
.format(variable)
"variable '{}' can not be found in variables mapping, "
"failed to export!".format(variable)
)
continue

30
httprunner/static/report_template.html
View File

@@ -279,15 +279,17 @@
{% endfor %}
<h3>Validators:</h3>
<div style="overflow: auto">
<table>
<div style="overflow: auto">
{% set validate_extractors = meta_data.validators.validate_extractor %}
{% if validate_extractors %}
<table>
<tr>
<th>check</th>
<th>comparator</th>
<th>expect value</th>
<th>actual value</th>
</tr>
{% for validator in meta_data.validators %}
{% for validator in validate_extractors %}
<tr>
{% if validator.check_result == "pass" %}
<td class="passed">
@@ -303,7 +305,27 @@
<td>{{validator.check_value | e}}</td>
</tr>
{% endfor %}
</table>
</table>
{% endif %}
{% set validate_script = meta_data.validators.validate_script %}
{% if validate_script %}
<table>
<tr>
<th>validate script</th><th>exception</th>
</tr>
<tr>
<td>{{validate_script.validate_script | safe}}</td>
{% if validate_script.check_result == "pass" %}
<td class="passed">
{% elif validate_script.check_result == "fail" %}
<td class="failed">
{% endif %}
{{validate_script.exception}}
</td>
</tr>
</table>
{% endif %}
</div>
<h3>Statistics:</h3>

51
httprunner/utils.py
View File

@@ -607,7 +607,7 @@ def omit_long_data(body, omit_len=512):
return omitted_body + appendix_str
def dump_json_file(json_data, pwd_dir_path, dump_file_name):
def dump_json_file(json_data, json_file_abs_path):
""" dump json data to file
"""
class PythonObjectEncoder(json.JSONEncoder):
@@ -617,14 +617,8 @@ def dump_json_file(json_data, pwd_dir_path, dump_file_name):
except TypeError:
return str(obj)
logs_dir_path = os.path.join(pwd_dir_path, "logs")
if not os.path.isdir(logs_dir_path):
os.makedirs(logs_dir_path)
dump_file_path = os.path.join(logs_dir_path, dump_file_name)
try:
with io.open(dump_file_path, 'w', encoding='utf-8') as outfile:
with io.open(json_file_abs_path, 'w', encoding='utf-8') as outfile:
if is_py2:
outfile.write(
unicode(json.dumps(
@@ -645,23 +639,44 @@ def dump_json_file(json_data, pwd_dir_path, dump_file_name):
cls=PythonObjectEncoder
)
msg = "dump file: {}".format(dump_file_path)
msg = "dump file: {}".format(json_file_abs_path)
logger.color_print(msg, "BLUE")
except TypeError as ex:
msg = "Failed to dump json file: {}\nReason: {}".format(dump_file_path, ex)
msg = "Failed to dump json file: {}\nReason: {}".format(json_file_abs_path, ex)
logger.color_print(msg, "RED")
def _prepare_dump_info(project_mapping, tag_name):
""" prepare dump file info.
def prepare_dump_json_file_abs_path(project_mapping, tag_name):
""" prepare dump json file absolute path.
"""
test_path = project_mapping.get("test_path") or "tests_mapping"
pwd_dir_path = project_mapping.get("PWD") or os.getcwd()
file_name, file_suffix = os.path.splitext(os.path.basename(test_path.rstrip("/")))
dump_file_name = "{}.{}.json".format(file_name, tag_name)
test_path = project_mapping.get("test_path")
return pwd_dir_path, dump_file_name
if not test_path:
# running passed in testcase/testsuite data structure
dump_file_name = "tests_mapping.{}.json".format(tag_name)
dumped_json_file_abs_path = os.path.join(pwd_dir_path, "logs", dump_file_name)
return dumped_json_file_abs_path
# both test_path and pwd_dir_path are absolute path
logs_dir_path = os.path.join(pwd_dir_path, "logs")
test_path_relative_path = test_path[len(pwd_dir_path)+1:]
if os.path.isdir(test_path):
file_foder_path = os.path.join(logs_dir_path, test_path_relative_path)
dump_file_name = "all.{}.json".format(tag_name)
else:
file_relative_folder_path, test_file = os.path.split(test_path_relative_path)
file_foder_path = os.path.join(logs_dir_path, file_relative_folder_path)
test_file_name, _file_suffix = os.path.splitext(test_file)
dump_file_name = "{}.{}.json".format(test_file_name, tag_name)
if not os.path.isdir(file_foder_path):
os.makedirs(file_foder_path)
dumped_json_file_abs_path = os.path.join(file_foder_path, dump_file_name)
return dumped_json_file_abs_path
def dump_logs(json_data, project_mapping, tag_name):
@@ -674,8 +689,8 @@ def dump_logs(json_data, project_mapping, tag_name):
tag_name (str): tag name, loaded/parsed/summary
"""
pwd_dir_path, dump_file_name = _prepare_dump_info(project_mapping, tag_name)
dump_json_file(json_data, pwd_dir_path, dump_file_name)
json_file_abs_path = prepare_dump_json_file_abs_path(project_mapping, tag_name)
dump_json_file(json_data, json_file_abs_path)
def get_python2_retire_msg():

360
httprunner/validator.py
View File

@@ -1,137 +1,8 @@
# encoding: utf-8
import collections
import io
import json
import os
import types
from httprunner import exceptions, logger
""" validate data format
TODO: refactor with JSON schema validate
"""
def is_testcase(data_structure):
""" check if data_structure is a testcase.
Args:
data_structure (dict): testcase should always be in the following data structure:
{
"config": {
"name": "desc1",
"variables": [], # optional
"request": {} # optional
},
"teststeps": [
test_dict1,
{ # test_dict2
'name': 'test step desc2',
'variables': [], # optional
'extract': [], # optional
'validate': [],
'request': {},
'function_meta': {}
}
]
}
Returns:
bool: True if data_structure is valid testcase, otherwise False.
"""
# TODO: replace with JSON schema validation
if not isinstance(data_structure, dict):
return False
if "teststeps" not in data_structure:
return False
if not isinstance(data_structure["teststeps"], list):
return False
return True
def is_testcases(data_structure):
""" check if data_structure is testcase or testcases list.
Args:
data_structure (dict): testcase(s) should always be in the following data structure:
{
"project_mapping": {
"PWD": "XXXXX",
"functions": {},
"env": {}
},
"testcases": [
{ # testcase data structure
"config": {
"name": "desc1",
"path": "testcase1_path",
"variables": [], # optional
},
"teststeps": [
# test data structure
{
'name': 'test step desc1',
'variables': [], # optional
'extract': [], # optional
'validate': [],
'request': {}
},
test_dict_2 # another test dict
]
},
testcase_dict_2 # another testcase dict
]
}
Returns:
bool: True if data_structure is valid testcase(s), otherwise False.
"""
if not isinstance(data_structure, dict):
return False
if "testcases" not in data_structure:
return False
testcases = data_structure["testcases"]
if not isinstance(testcases, list):
return False
for item in testcases:
if not is_testcase(item):
return False
return True
def is_testcase_path(path):
""" check if path is testcase path or path list.
Args:
path (str/list): file path or file path list.
Returns:
bool: True if path is valid file path or path list, otherwise False.
"""
if not isinstance(path, (str, list)):
return False
if isinstance(path, list):
for p in path:
if not is_testcase_path(p):
return False
if isinstance(path, str):
if not os.path.exists(path):
return False
return True
from httprunner import exceptions, logger, parser
###############################################################################
@@ -159,13 +30,13 @@ def get_uniform_comparator(comparator):
return "length_equals"
elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]:
return "length_greater_than"
elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \
"count_greater_than_or_equals"]:
elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals",
"count_greater_than_or_equals"]:
return "length_greater_than_or_equals"
elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]:
return "length_less_than"
elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \
"count_less_than_or_equals"]:
elif comparator in ["len_le", "count_le", "length_less_than_or_equals",
"count_less_than_or_equals"]:
return "length_less_than_or_equals"
else:
return comparator
@@ -237,14 +108,14 @@ def _convert_validators_to_mapping(validators):
Examples:
>>> validators = [
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": {"b": 1}, "expect": 200, "comparator": "eq"}
]
>>> _convert_validators_to_mapping(validators)
{
("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
}
{"check": "v1", "expect": 201, "comparator": "eq"},
{"check": {"b": 1}, "expect": 200, "comparator": "eq"}
]
>>> print(_convert_validators_to_mapping(validators))
{
("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
}
"""
validators_mapping = {}
@@ -298,50 +169,189 @@ def extend_validators(raw_validators, override_validators):
return list(def_validators_mapping.values())
###############################################################################
## validate varibles and functions
###############################################################################
class Validator(object):
"""Validate tests
Attributes:
validation_results (dict): store validation results,
including validate_extractor and validate_script.
def is_function(item):
""" Takes item object, returns True if it is a function.
"""
return isinstance(item, types.FunctionType)
def __init__(self, session_context, resp_obj):
""" initialize a Validator for each teststep (API request)
def is_variable(tup):
""" Takes (name, object) tuple, returns True if it is a variable.
"""
name, item = tup
if callable(item):
# function or class
return False
Args:
session_context: HttpRunner session context
resp_obj: ResponseObject instance
"""
self.session_context = session_context
self.resp_obj = resp_obj
self.validation_results = {}
if isinstance(item, types.ModuleType):
# imported module
return False
def __eval_validator_check(self, check_item):
""" evaluate check item in validator.
if name.startswith("_"):
# private property
return False
Args:
check_item: check_item should only be the following 5 formats:
1, variable reference, e.g. $token
2, function reference, e.g. ${is_status_code_200($status_code)}
3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
4, string joined by delimiter. e.g. "status_code", "headers.content-type"
5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
return True
"""
if isinstance(check_item, (dict, list)) \
or isinstance(check_item, parser.LazyString):
# format 1/2/3
check_value = self.session_context.eval_content(check_item)
else:
# format 4/5
check_value = self.resp_obj.extract_field(check_item)
return check_value
def validate_json_file(file_list):
""" validate JSON testcase format
"""
for json_file in set(file_list):
if not json_file.endswith(".json"):
logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file))
continue
def __eval_validator_expect(self, expect_item):
""" evaluate expect item in validator.
logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN")
Args:
expect_item: expect_item should only be in 2 types:
1, variable reference, e.g. $expect_status_code
2, actual value, e.g. 200
"""
expect_value = self.session_context.eval_content(expect_item)
return expect_value
def validate_script(self, script):
""" make validation with python script
"""
validator_dict = {
"validate_script": "<br/>".join(script),
"check_result": "fail",
"exception": ""
}
script = "\n ".join(script)
code = """
# encoding: utf-8
try:
{}
except Exception as ex:
import traceback
import sys
_type, _value, _tb = sys.exc_info()
# filename, lineno, name, line
_, _lineno, _, line_content = traceback.extract_tb(_tb, 1)[0]
line_no = _lineno - 4
c_exception = _type.__name__ + "\\n"
c_exception += "\\tError line number: " + str(line_no) + "\\n"
c_exception += "\\tError line content: " + str(line_content) + "\\n"
if _value.args:
c_exception += "\\tError description: " + str(_value)
else:
c_exception += "\\tError description: " + _type.__name__
raise _type(c_exception)
""".format(script)
variables = {
"status_code": self.resp_obj.status_code,
"response_json": self.resp_obj.json,
"response": self.resp_obj
}
variables.update(self.session_context.test_variables_mapping)
try:
code = compile(code, '<string>', 'exec')
exec(code, variables)
validator_dict["check_result"] = "pass"
return validator_dict, ""
except Exception as ex:
validator_dict["check_result"] = "fail"
validator_dict["exception"] = "<br/>".join(str(ex).splitlines())
return validator_dict, str(ex)
def validate(self, validators):
""" make validation with comparators
"""
self.validation_results = {}
if not validators:
return
logger.log_debug("start to validate.")
validate_pass = True
failures = []
for validator in validators:
if isinstance(validator, dict) and validator.get("type") == "python_script":
validator_dict, ex = self.validate_script(validator["script"])
if ex:
validate_pass = False
failures.append(ex)
self.validation_results["validate_script"] = validator_dict
continue
if "validate_extractor" not in self.validation_results:
self.validation_results["validate_extractor"] = []
# validator should be LazyFunction object
if not isinstance(validator, parser.LazyFunction):
raise exceptions.ValidationFailure(
"validator should be parsed first: {}".format(validators))
# evaluate validator args with context variable mapping.
validator_args = validator.get_args()
check_item, expect_item = validator_args
check_value = self.__eval_validator_check(check_item)
expect_value = self.__eval_validator_expect(expect_item)
validator.update_args([check_value, expect_value])
comparator = validator.func_name
validator_dict = {
"comparator": comparator,
"check": check_item,
"check_value": check_value,
"expect": expect_item,
"expect_value": expect_value
}
validate_msg = "\nvalidate: {} {} {}({})".format(
check_item,
comparator,
expect_value,
type(expect_value).__name__
)
with io.open(json_file) as stream:
try:
json.load(stream)
except ValueError as e:
raise SystemExit(e)
validator.to_value(self.session_context.test_variables_mapping)
validator_dict["check_result"] = "pass"
validate_msg += "\t==> pass"
logger.log_debug(validate_msg)
except (AssertionError, TypeError):
validate_pass = False
validator_dict["check_result"] = "fail"
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__,
comparator,
expect_value,
type(expect_value).__name__
)
logger.log_error(validate_msg)
failures.append(validate_msg)
print("OK")
self.validation_results["validate_extractor"].append(validator_dict)
# restore validator args, in case of running multiple times
validator.update_args(validator_args)
if not validate_pass:
failures_string = "\n".join([failure for failure in failures])
raise exceptions.ValidationFailure(failures_string)

2
pyproject.toml
View File

@@ -1,6 +1,6 @@
[tool.poetry]
name = "httprunner"
version = "2.3.2"
version = "2.4.0"
description = "One-stop solution for HTTP(S) testing."
license = "Apache-2.0"
readme = "README.md"

9
tests/debugtalk.py
View File

@@ -127,6 +127,15 @@ def alter_response(response):
"key": 123
}
def alter_response_302(response):
response.status_code = 500
response.headers["Content-Type"] = "html/text"
response.text = "abcdef"
response.new_attribute = "new_attribute_value"
response.new_attribute_dict = {
"key": 123
}
def alter_response_error(response):
# NameError

10
tests/httpbin/a.b.c/rpc.yml Normal file
View File

@@ -0,0 +1,10 @@
name: rpc api
base_url: http://httpbin.org
variables:
expected_status_code: 200
request:
url: /headers
method: GET
validate:
- eq: ["status_code", $expected_status_code]
- eq: [content.headers.Host, "httpbin.org"]

13
tests/httpbin/api/302_redirect_teardown_hook.yml Normal file
View File

@@ -0,0 +1,13 @@
name: 302 redirect
request:
url: https://httpbin.org/redirect-to
params:
url: https://github.com
status_code: 302
method: GET
verify: False
teardown_hooks:
- ${alter_response_302($response)}
validate:
- eq: ["status_code", 500]
- eq: ["text","abcdef"]

31
tests/httpbin/validate.yml
View File

@@ -1,13 +1,34 @@
- config:
name: basic test with httpbin
request:
base_url: http://httpbin.org/
base_url: http://httpbin.org/
- test:
name: headers
name: validate response with json path
request:
url: /headers
url: /get
params:
a: 1
b: 2
method: GET
validate:
- eq: ["status_code", 200]
- assert_status_code_is_200: ["status_code"]
- eq: ["json.args.a", '1']
- eq: ["json.args.b", '2']
validate_script:
- "assert status_code == 200"
- test:
name: validate response with python script
request:
url: /get
params:
a: 1
b: 2
method: GET
validate:
- eq: ["status_code", 200]
validate_script:
- "assert status_code == 201"
- "a = response_json.get('args').get('a')"
- "assert a == '1'"

29
tests/test_api.py
View File

@@ -289,6 +289,10 @@ class TestHttpRunner(ApiServerUnittest):
self.assertEqual(summary["stat"]["testcases"]["total"], 2)
self.assertEqual(summary["stat"]["teststeps"]["total"], 4)
def test_validate_script(self):
summary = self.runner.run("tests/httpbin/validate.yml")
self.assertFalse(summary["success"])
def test_run_httprunner_with_hooks(self):
testcase_file_path = os.path.join(
os.getcwd(), 'tests/httpbin/hooks.yml')
@@ -419,6 +423,19 @@ class TestHttpRunner(ApiServerUnittest):
self.assertEqual(req_resp_data[0]["response"]["status_code"], 302)
self.assertEqual(req_resp_data[1]["response"]["status_code"], 200)
def test_request_302_logs_teardown_hook(self):
path = "tests/httpbin/api/302_redirect_teardown_hook.yml"
summary = self.runner.run(path)
self.assertTrue(summary["success"])
self.assertEqual(summary["stat"]["testcases"]["total"], 1)
self.assertEqual(summary["stat"]["teststeps"]["total"], 1)
self.assertEqual(summary["stat"]["teststeps"]["successes"], 1)
req_resp_data = summary["details"][0]["records"][0]["meta_datas"]["data"]
self.assertEqual(len(req_resp_data), 2)
self.assertEqual(req_resp_data[0]["response"]["status_code"], 302)
self.assertEqual(req_resp_data[1]["response"]["status_code"], 500)
def test_request_with_params(self):
path = "tests/httpbin/api/302_redirect.yml"
summary = self.runner.run(path)
@@ -437,13 +454,17 @@ class TestHttpRunner(ApiServerUnittest):
def test_run_api_folder(self):
api_folder = "tests/httpbin/api/"
summary = self.runner.run(api_folder)
print(summary["stat"]["testcases"]["total"])
print(len(summary["details"]))
self.assertTrue(summary["success"])
self.assertEqual(summary["stat"]["testcases"]["total"], 2)
self.assertEqual(summary["stat"]["teststeps"]["total"], 2)
self.assertEqual(summary["stat"]["teststeps"]["successes"], 2)
self.assertEqual(len(summary["details"]), 2)
self.assertEqual(summary["stat"]["testcases"]["total"], 3)
self.assertEqual(summary["stat"]["teststeps"]["total"], 3)
self.assertEqual(summary["stat"]["teststeps"]["successes"], 3)
self.assertEqual(len(summary["details"]), 3)
self.assertEqual(summary["details"][0]["stat"]["total"], 1)
self.assertEqual(summary["details"][1]["stat"]["total"], 1)
self.assertEqual(summary["details"][2]["stat"]["total"], 1)
def test_run_testcase_hardcode(self):
for testcase_file_path in self.testcase_file_path_list:

70
tests/test_loader.py
View File

@@ -476,3 +476,73 @@ class TestSuiteLoader(unittest.TestCase):
api_file_path = os.path.join(os.getcwd(), "tests", "api", "get_token.yml")
self.assertIn(api_file_path, self.tests_def_mapping["api"])
self.assertEqual(self.project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH")
def test_is_function(self):
func = lambda x: x + 1
self.assertTrue(loader.is_function(func))
self.assertTrue(loader.is_function(loader.is_testcase))
def test_is_testcases(self):
data_structure = "path/to/file"
self.assertFalse(loader.is_testcases(data_structure))
data_structure = ["path/to/file1", "path/to/file2"]
self.assertFalse(loader.is_testcases(data_structure))
data_structure = {
"project_mapping": {
"PWD": "XXXXX",
"functions": {},
"env": {}
},
"testcases": [
{ # testcase data structure
"config": {
"name": "desc1",
"path": "testcase1_path",
"variables": [], # optional
},
"teststeps": [
# test data structure
{
'name': 'test step desc1',
'variables': [], # optional
'extract': [], # optional
'validate': [],
'request': {}
},
# test_dict2 # another test dict
]
},
# testcase_dict_2 # another testcase dict
]
}
self.assertTrue(loader.is_testcases(data_structure))
data_structure = [
{
"name": "desc1",
"config": {},
"api": {},
"testcases": ["testcase11", "testcase12"]
},
{
"name": "desc2",
"config": {},
"api": {},
"testcases": ["testcase21", "testcase22"]
}
]
self.assertTrue(data_structure)
def test_is_variable(self):
var1 = 123
var2 = "abc"
self.assertTrue(loader.is_variable(("var1", var1)))
self.assertTrue(loader.is_variable(("var2", var2)))
__var = 123
self.assertFalse(loader.is_variable(("__var", __var)))
func = lambda x: x + 1
self.assertFalse(loader.is_variable(("func", func)))
self.assertFalse(loader.is_variable(("unittest", unittest)))

34
tests/test_utils.py
View File

@@ -275,3 +275,37 @@ class TestUtils(ApiServerUnittest):
"d": [4, 5]
}
utils.print_info(info_mapping)
def test_prepare_dump_json_file_path_for_folder(self):
# hrun tests/httpbin/a.b.c/ --save-tests
project_working_directory = os.path.join(os.getcwd(), "tests")
project_mapping = {
"PWD": project_working_directory,
"test_path": os.path.join(os.getcwd(), "tests", "httpbin", "a.b.c")
}
self.assertEqual(
utils.prepare_dump_json_file_abs_path(project_mapping, "loaded"),
os.path.join(project_working_directory, "logs", "httpbin/a.b.c/all.loaded.json")
)
def test_prepare_dump_json_file_path_for_file(self):
# hrun tests/httpbin/a.b.c/rpc.yml --save-tests
project_working_directory = os.path.join(os.getcwd(), "tests")
project_mapping = {
"PWD": project_working_directory,
"test_path": os.path.join(os.getcwd(), "tests", "httpbin", "a.b.c", "rpc.yml")
}
self.assertEqual(
utils.prepare_dump_json_file_abs_path(project_mapping, "loaded"),
os.path.join(project_working_directory, "logs", "httpbin/a.b.c/rpc.loaded.json")
)
def test_prepare_dump_json_file_path_for_passed_testcase(self):
project_working_directory = os.path.join(os.getcwd(), "tests")
project_mapping = {
"PWD": project_working_directory
}
self.assertEqual(
utils.prepare_dump_json_file_abs_path(project_mapping, "loaded"),
os.path.join(project_working_directory, "logs", "tests_mapping.loaded.json")
)

71
tests/test_validator.py
View File

@@ -5,76 +5,6 @@ from httprunner import validator
class TestValidator(unittest.TestCase):
def test_is_testcases(self):
data_structure = "path/to/file"
self.assertFalse(validator.is_testcases(data_structure))
data_structure = ["path/to/file1", "path/to/file2"]
self.assertFalse(validator.is_testcases(data_structure))
data_structure = {
"project_mapping": {
"PWD": "XXXXX",
"functions": {},
"env": {}
},
"testcases": [
{ # testcase data structure
"config": {
"name": "desc1",
"path": "testcase1_path",
"variables": [], # optional
},
"teststeps": [
# test data structure
{
'name': 'test step desc1',
'variables': [], # optional
'extract': [], # optional
'validate': [],
'request': {}
},
# test_dict2 # another test dict
]
},
# testcase_dict_2 # another testcase dict
]
}
self.assertTrue(validator.is_testcases(data_structure))
data_structure = [
{
"name": "desc1",
"config": {},
"api": {},
"testcases": ["testcase11", "testcase12"]
},
{
"name": "desc2",
"config": {},
"api": {},
"testcases": ["testcase21", "testcase22"]
}
]
self.assertTrue(data_structure)
def test_is_variable(self):
var1 = 123
var2 = "abc"
self.assertTrue(validator.is_variable(("var1", var1)))
self.assertTrue(validator.is_variable(("var2", var2)))
__var = 123
self.assertFalse(validator.is_variable(("__var", __var)))
func = lambda x: x + 1
self.assertFalse(validator.is_variable(("func", func)))
self.assertFalse(validator.is_variable(("unittest", unittest)))
def test_is_function(self):
func = lambda x: x + 1
self.assertTrue(validator.is_function(func))
self.assertTrue(validator.is_function(validator.is_testcase))
def test_get_uniform_comparator(self):
self.assertEqual(validator.get_uniform_comparator("eq"), "equals")
self.assertEqual(validator.get_uniform_comparator("=="), "equals")
@@ -117,7 +47,6 @@ class TestValidator(unittest.TestCase):
{"check": "status_code", "comparator": "equals", "expect": 201}
)
def test_extend_validators(self):
def_validators = [
{'eq': ['v1', 200]},