mirror of
https://github.com/httprunner/httprunner.git
synced 2026-06-15 12:49:39 +08:00
refactor: split validate from context, move to validator
This commit is contained in:
debugtalk
@@ -2,136 +2,9 @@
|
||||
import collections
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import types
|
||||
|
||||
from httprunner import exceptions, logger
|
||||
|
||||
""" validate data format
|
||||
TODO: refactor with JSON schema validate
|
||||
"""
|
||||
|
||||
|
||||
def is_testcase(data_structure):
|
||||
""" check if data_structure is a testcase.
|
||||
|
||||
Args:
|
||||
data_structure (dict): testcase should always be in the following data structure:
|
||||
|
||||
{
|
||||
"config": {
|
||||
"name": "desc1",
|
||||
"variables": [], # optional
|
||||
"request": {} # optional
|
||||
},
|
||||
"teststeps": [
|
||||
test_dict1,
|
||||
{ # test_dict2
|
||||
'name': 'test step desc2',
|
||||
'variables': [], # optional
|
||||
'extract': [], # optional
|
||||
'validate': [],
|
||||
'request': {},
|
||||
'function_meta': {}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Returns:
|
||||
bool: True if data_structure is valid testcase, otherwise False.
|
||||
|
||||
"""
|
||||
# TODO: replace with JSON schema validation
|
||||
if not isinstance(data_structure, dict):
|
||||
return False
|
||||
|
||||
if "teststeps" not in data_structure:
|
||||
return False
|
||||
|
||||
if not isinstance(data_structure["teststeps"], list):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_testcases(data_structure):
|
||||
""" check if data_structure is testcase or testcases list.
|
||||
|
||||
Args:
|
||||
data_structure (dict): testcase(s) should always be in the following data structure:
|
||||
{
|
||||
"project_mapping": {
|
||||
"PWD": "XXXXX",
|
||||
"functions": {},
|
||||
"env": {}
|
||||
},
|
||||
"testcases": [
|
||||
{ # testcase data structure
|
||||
"config": {
|
||||
"name": "desc1",
|
||||
"path": "testcase1_path",
|
||||
"variables": [], # optional
|
||||
},
|
||||
"teststeps": [
|
||||
# test data structure
|
||||
{
|
||||
'name': 'test step desc1',
|
||||
'variables': [], # optional
|
||||
'extract': [], # optional
|
||||
'validate': [],
|
||||
'request': {}
|
||||
},
|
||||
test_dict_2 # another test dict
|
||||
]
|
||||
},
|
||||
testcase_dict_2 # another testcase dict
|
||||
]
|
||||
}
|
||||
|
||||
Returns:
|
||||
bool: True if data_structure is valid testcase(s), otherwise False.
|
||||
|
||||
"""
|
||||
if not isinstance(data_structure, dict):
|
||||
return False
|
||||
|
||||
if "testcases" not in data_structure:
|
||||
return False
|
||||
|
||||
testcases = data_structure["testcases"]
|
||||
if not isinstance(testcases, list):
|
||||
return False
|
||||
|
||||
for item in testcases:
|
||||
if not is_testcase(item):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_testcase_path(path):
|
||||
""" check if path is testcase path or path list.
|
||||
|
||||
Args:
|
||||
path (str/list): file path or file path list.
|
||||
|
||||
Returns:
|
||||
bool: True if path is valid file path or path list, otherwise False.
|
||||
|
||||
"""
|
||||
if not isinstance(path, (str, list)):
|
||||
return False
|
||||
|
||||
if isinstance(path, list):
|
||||
for p in path:
|
||||
if not is_testcase_path(p):
|
||||
return False
|
||||
|
||||
if isinstance(path, str):
|
||||
if not os.path.exists(path):
|
||||
return False
|
||||
|
||||
return True
|
||||
from httprunner import exceptions, logger, parser
|
||||
|
||||
|
||||
###############################################################################
|
||||
@@ -345,3 +218,191 @@ def validate_json_file(file_list):
|
||||
raise SystemExit(e)
|
||||
|
||||
print("OK")
|
||||
|
||||
|
||||
class Validator(object):
|
||||
"""Validate tests
|
||||
|
||||
Attributes:
|
||||
validation_results (dict): store validation results,
|
||||
including validate_extractor and validate_script.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, session_context, resp_obj):
|
||||
""" initialize a Validator for each teststep (API request)
|
||||
|
||||
Args:
|
||||
session_context: HttpRunner session context
|
||||
resp_obj: ResponseObject instance
|
||||
"""
|
||||
self.session_context = session_context
|
||||
self.resp_obj = resp_obj
|
||||
self.validation_results = {}
|
||||
|
||||
def __eval_validator_check(self, check_item):
|
||||
""" evaluate check item in validator.
|
||||
|
||||
Args:
|
||||
check_item: check_item should only be the following 5 formats:
|
||||
1, variable reference, e.g. $token
|
||||
2, function reference, e.g. ${is_status_code_200($status_code)}
|
||||
3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"}
|
||||
4, string joined by delimiter. e.g. "status_code", "headers.content-type"
|
||||
5, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
|
||||
|
||||
"""
|
||||
if isinstance(check_item, (dict, list)) \
|
||||
or isinstance(check_item, parser.LazyString):
|
||||
# format 1/2/3
|
||||
check_value = self.session_context.eval_content(check_item)
|
||||
else:
|
||||
# format 4/5
|
||||
check_value = self.resp_obj.extract_field(check_item)
|
||||
|
||||
return check_value
|
||||
|
||||
def __eval_validator_expect(self, expect_item):
|
||||
""" evaluate expect item in validator.
|
||||
|
||||
Args:
|
||||
expect_item: expect_item should only be in 2 types:
|
||||
1, variable reference, e.g. $expect_status_code
|
||||
2, actual value, e.g. 200
|
||||
|
||||
"""
|
||||
expect_value = self.session_context.eval_content(expect_item)
|
||||
return expect_value
|
||||
|
||||
def validate_script(self, script):
|
||||
""" make validation with python script
|
||||
"""
|
||||
validator_dict = {
|
||||
"validate_script": "<br/>".join(script),
|
||||
"check_result": "fail",
|
||||
"exception": ""
|
||||
}
|
||||
|
||||
script = "\n ".join(script)
|
||||
code = """
|
||||
# encoding: utf-8
|
||||
|
||||
try:
|
||||
{}
|
||||
except Exception as ex:
|
||||
import traceback
|
||||
import sys
|
||||
_type, _value, _tb = sys.exc_info()
|
||||
# filename, lineno, name, line
|
||||
_, _lineno, _, line_content = traceback.extract_tb(_tb, 1)[0]
|
||||
|
||||
line_no = _lineno - 4
|
||||
|
||||
c_exception = _type.__name__ + "\\n"
|
||||
c_exception += "\\tError line number: " + str(line_no) + "\\n"
|
||||
c_exception += "\\tError line content: " + str(line_content) + "\\n"
|
||||
|
||||
if _value.args:
|
||||
c_exception += "\\tError description: " + str(_value)
|
||||
else:
|
||||
c_exception += "\\tError description: " + _type.__name__
|
||||
|
||||
raise _type(c_exception)
|
||||
""".format(script)
|
||||
variables = {
|
||||
"status_code": self.resp_obj.status_code,
|
||||
"response_json": self.resp_obj.json,
|
||||
"response": self.resp_obj
|
||||
}
|
||||
variables.update(self.session_context.test_variables_mapping)
|
||||
|
||||
try:
|
||||
code = compile(code, '<string>', 'exec')
|
||||
exec(code, variables)
|
||||
validator_dict["check_result"] = "pass"
|
||||
return validator_dict, ""
|
||||
except Exception as ex:
|
||||
validator_dict["check_result"] = "fail"
|
||||
validator_dict["exception"] = "<br/>".join(str(ex).splitlines())
|
||||
return validator_dict, str(ex)
|
||||
|
||||
def validate(self, validators):
|
||||
""" make validation with comparators
|
||||
"""
|
||||
self.validation_results = {}
|
||||
if not validators:
|
||||
return
|
||||
|
||||
logger.log_debug("start to validate.")
|
||||
|
||||
validate_pass = True
|
||||
failures = []
|
||||
|
||||
for validator in validators:
|
||||
|
||||
if isinstance(validator, dict) and validator.get("type") == "python_script":
|
||||
validator_dict, ex = self.validate_script(validator["script"])
|
||||
if ex:
|
||||
validate_pass = False
|
||||
failures.append(ex)
|
||||
|
||||
self.validation_results["validate_script"] = validator_dict
|
||||
continue
|
||||
|
||||
if "validate_extractor" not in self.validation_results:
|
||||
self.validation_results["validate_extractor"] = []
|
||||
|
||||
# validator should be LazyFunction object
|
||||
if not isinstance(validator, parser.LazyFunction):
|
||||
raise exceptions.ValidationFailure(
|
||||
"validator should be parsed first: {}".format(validators))
|
||||
|
||||
# evaluate validator args with context variable mapping.
|
||||
validator_args = validator.get_args()
|
||||
check_item, expect_item = validator_args
|
||||
check_value = self.__eval_validator_check(check_item)
|
||||
expect_value = self.__eval_validator_expect(expect_item)
|
||||
validator.update_args([check_value, expect_value])
|
||||
|
||||
comparator = validator.func_name
|
||||
validator_dict = {
|
||||
"comparator": comparator,
|
||||
"check": check_item,
|
||||
"check_value": check_value,
|
||||
"expect": expect_item,
|
||||
"expect_value": expect_value
|
||||
}
|
||||
validate_msg = "\nvalidate: {} {} {}({})".format(
|
||||
check_item,
|
||||
comparator,
|
||||
expect_value,
|
||||
type(expect_value).__name__
|
||||
)
|
||||
|
||||
try:
|
||||
validator.to_value(self.session_context.test_variables_mapping)
|
||||
validator_dict["check_result"] = "pass"
|
||||
validate_msg += "\t==> pass"
|
||||
logger.log_debug(validate_msg)
|
||||
except (AssertionError, TypeError):
|
||||
validate_pass = False
|
||||
validator_dict["check_result"] = "fail"
|
||||
validate_msg += "\t==> fail"
|
||||
validate_msg += "\n{}({}) {} {}({})".format(
|
||||
check_value,
|
||||
type(check_value).__name__,
|
||||
comparator,
|
||||
expect_value,
|
||||
type(expect_value).__name__
|
||||
)
|
||||
logger.log_error(validate_msg)
|
||||
failures.append(validate_msg)
|
||||
|
||||
self.validation_results["validate_extractor"].append(validator_dict)
|
||||
|
||||
# restore validator args, in case of running multiple times
|
||||
validator.update_args(validator_args)
|
||||
|
||||
if not validate_pass:
|
||||
failures_string = "\n".join([failure for failure in failures])
|
||||
raise exceptions.ValidationFailure(failures_string)
|
||||
|
||||
Reference in New Issue
Block a user