diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5a937ad0..04570db4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 2.4.0 (2019-12-04) + +**Added** + +- feat: validate with python script, ref #773 + ## 2.3.3 (2019-12-04) **Fixed** diff --git a/httprunner/__init__.py b/httprunner/__init__.py index 4d5c880e..cb8ce2a1 100644 --- a/httprunner/__init__.py +++ b/httprunner/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.3.3" +__version__ = "2.4.0" __description__ = "One-stop solution for HTTP(S) testing." __all__ = ["__version__", "__description__"] diff --git a/httprunner/context.py b/httprunner/context.py index 980f44a9..27b3406c 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -13,11 +13,13 @@ class SessionContext(object): >>> context.update_session_variables(variables) """ + def __init__(self, variables=None): variables_mapping = utils.ensure_mapping_format(variables or {}) self.session_variables_mapping = parser.parse_variables_mapping(variables_mapping) + self.test_variables_mapping = {} self.init_test_variables() - self.validation_results = [] + self.validation_results = {} def init_test_variables(self, variables_mapping=None): """ init test variables, called when each test(api) starts. @@ -77,7 +79,7 @@ class SessionContext(object): """ if isinstance(check_item, (dict, list)) \ - or isinstance(check_item, parser.LazyString): + or isinstance(check_item, parser.LazyString): # format 1/2/3 check_value = self.eval_content(check_item) else: @@ -101,7 +103,7 @@ class SessionContext(object): def validate(self, validators, resp_obj): """ make validation with comparators """ - self.validation_results = [] + self.validation_results = {} if not validators: return @@ -111,6 +113,19 @@ class SessionContext(object): failures = [] for validator in validators: + + if isinstance(validator, dict) and validator.get("type") == "python_script": + validator_dict, ex = self.validate_script(validator["script"], resp_obj) + if ex: + validate_pass = False + failures.append(ex) + + self.validation_results["validate_script"] = validator_dict + continue + + if "validate_extractor" not in self.validation_results: + self.validation_results["validate_extractor"] = [] + # validator should be LazyFunction object if not isinstance(validator, parser.LazyFunction): raise exceptions.ValidationFailure( @@ -160,7 +175,7 @@ class SessionContext(object): logger.log_error(validate_msg) failures.append(validate_msg) - self.validation_results.append(validator_dict) + self.validation_results["validate_extractor"].append(validator_dict) # restore validator args, in case of running multiple times validator.update_args(validator_args) @@ -168,3 +183,55 @@ class SessionContext(object): if not validate_pass: failures_string = "\n".join([failure for failure in failures]) raise exceptions.ValidationFailure(failures_string) + + def validate_script(self, script, resp_obj): + """ make validation with python script + """ + validator_dict = { + "validate_script": "
".join(script), + "check_result": "fail", + "exception": "" + } + + script = "\n ".join(script) + code = f""" +# encoding: utf-8 + +try: + {script} +except Exception as ex: + import traceback + import sys + _type, _value, _tb = sys.exc_info() + # filename, lineno, name, line + _, _lineno, _, line_content = traceback.extract_tb(_tb, 1)[0] + + line_no = _lineno - 4 + + c_exception = _type.__name__ + "\\n" + c_exception += "\\tError line number: " + str(line_no) + "\\n" + c_exception += "\\tError line content: " + str(line_content) + "\\n" + + if _value.args: + c_exception += "\\tError description: " + str(_value) + else: + c_exception += "\\tError description: " + _type.__name__ + + raise _type(c_exception) +""" + variables = { + "status_code": resp_obj.status_code, + "response_json": resp_obj.json, + "response": resp_obj + } + variables.update(self.test_variables_mapping) + + try: + code = compile(code, '', 'exec') + exec(code, variables) + validator_dict["check_result"] = "pass" + return validator_dict, "" + except Exception as ex: + validator_dict["check_result"] = "fail" + validator_dict["exception"] = "
".join(str(ex).splitlines()) + return validator_dict, str(ex) diff --git a/httprunner/runner.py b/httprunner/runner.py index 13f0a9c9..116a58f3 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -62,7 +62,6 @@ class Runner(object): """ self.verify = config.get("verify", True) self.export = config.get("export") or config.get("output", []) - self.validation_results = [] config_variables = config.get("variables", {}) # testcase setup hooks @@ -86,7 +85,6 @@ class Runner(object): if not isinstance(self.http_client_session, HttpSession): return - self.validation_results = [] self.http_client_session.init_meta_data() def __get_test_data(self): @@ -96,7 +94,7 @@ class Runner(object): return meta_data = self.http_client_session.meta_data - meta_data["validators"] = self.validation_results + meta_data["validators"] = self.session_context.validation_results return meta_data def _handle_skip_feature(self, test_dict): @@ -244,7 +242,8 @@ class Runner(object): raise exceptions.ParamsError(err_msg) logger.log_info("{method} {url}".format(method=method, url=parsed_url)) - logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request)) + logger.log_debug( + "request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request)) # request resp = self.http_client_session.request( @@ -268,10 +267,19 @@ class Runner(object): self.session_context.update_session_variables(extracted_variables_mapping) # validate + # TODO: split validate from context validators = test_dict.get("validate") or test_dict.get("validators") or [] + validate_script = test_dict.get("validate_script", []) + if validate_script: + validators.append({ + "type": "python_script", + "script": validate_script + }) + try: self.session_context.validate(validators, resp_obj) - except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure): + except (exceptions.ParamsError, + exceptions.ValidationFailure, exceptions.ExtractFailure): err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32) # log request @@ -294,9 +302,6 @@ class Runner(object): raise - finally: - self.validation_results = self.session_context.validation_results - def _run_testcase(self, testcase_dict): """ run single testcase. """ diff --git a/httprunner/static/report_template.html b/httprunner/static/report_template.html index 633b97c5..209c18ae 100644 --- a/httprunner/static/report_template.html +++ b/httprunner/static/report_template.html @@ -279,15 +279,17 @@ {% endfor %}

Validators:

-
- +
+ {% set validate_extractors = meta_data.validators.validate_extractor %} + {% if validate_extractors %} +
- {% for validator in meta_data.validators %} + {% for validator in validate_extractors %} {% if validator.check_result == "pass" %} {% endfor %} -
check comparator expect value actual value
@@ -303,7 +305,27 @@ {{validator.check_value | e}}
+ + {% endif %} + + {% set validate_script = meta_data.validators.validate_script %} + {% if validate_script %} + + + + + + + {% if validate_script.check_result == "pass" %} + + +
validate scriptexception
{{validate_script.validate_script | safe}} + {% elif validate_script.check_result == "fail" %} + + {% endif %} + {{validate_script.exception}} +
+ {% endif %}

Statistics:

diff --git a/pyproject.toml b/pyproject.toml index 29385917..6b14c7d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "httprunner" -version = "2.3.3" +version = "2.4.0" description = "One-stop solution for HTTP(S) testing." license = "Apache-2.0" readme = "README.md" diff --git a/tests/httpbin/validate.yml b/tests/httpbin/validate.yml index 310b826b..0be60af8 100644 --- a/tests/httpbin/validate.yml +++ b/tests/httpbin/validate.yml @@ -1,13 +1,34 @@ - config: name: basic test with httpbin - request: - base_url: http://httpbin.org/ + base_url: http://httpbin.org/ - test: - name: headers + name: validate response with json path request: - url: /headers + url: /get + params: + a: 1 + b: 2 method: GET validate: - eq: ["status_code", 200] - - assert_status_code_is_200: ["status_code"] + - eq: ["json.args.a", '1'] + - eq: ["json.args.b", '2'] + validate_script: + - "assert status_code == 200" + + +- test: + name: validate response with python script + request: + url: /get + params: + a: 1 + b: 2 + method: GET + validate: + - eq: ["status_code", 200] + validate_script: + - "assert status_code == 201" + - "a = response_json.get('args').get('a')" + - "assert a == '1'" diff --git a/tests/test_api.py b/tests/test_api.py index d4f2e125..9e61f179 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -289,6 +289,10 @@ class TestHttpRunner(ApiServerUnittest): self.assertEqual(summary["stat"]["testcases"]["total"], 2) self.assertEqual(summary["stat"]["teststeps"]["total"], 4) + def test_validate_script(self): + summary = self.runner.run("tests/httpbin/validate.yml") + self.assertFalse(summary["success"]) + def test_run_httprunner_with_hooks(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/httpbin/hooks.yml')