From 9aef959d7c55d07b4f9cd31d70858d5a3f667064 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 19 Apr 2020 17:37:23 +0800 Subject: [PATCH] change v3: add basic validation --- .../{hardcode.py => hardcode_test.py} | 3 +- .../v3/{parser/__init__.py => parser.py} | 7 +- httprunner/v3/response.py | 13 ++ .../v3/{runner/__init__.py => runner.py} | 25 ++- httprunner/v3/validator.py | 145 ++++++++++++++++++ poetry.lock | 14 +- pyproject.toml | 1 + 7 files changed, 200 insertions(+), 8 deletions(-) rename examples/postman_echo/request_methods/{hardcode.py => hardcode_test.py} (95%) rename httprunner/v3/{parser/__init__.py => parser.py} (97%) create mode 100644 httprunner/v3/response.py rename httprunner/v3/{runner/__init__.py => runner.py} (62%) create mode 100644 httprunner/v3/validator.py diff --git a/examples/postman_echo/request_methods/hardcode.py b/examples/postman_echo/request_methods/hardcode_test.py similarity index 95% rename from examples/postman_echo/request_methods/hardcode.py rename to examples/postman_echo/request_methods/hardcode_test.py index d1129ab6..52f7899b 100644 --- a/examples/postman_echo/request_methods/hardcode.py +++ b/examples/postman_echo/request_methods/hardcode_test.py @@ -24,7 +24,8 @@ class TestCaseRequestMethodsHardcode(TestCaseRunner): } }, "validate": [ - {"eq": ["status_code", 200]} + {"eq": ["status_code", 200]}, + {"eq": ["headers.Server", "nginx"]} ] }), TestStep(**{ diff --git a/httprunner/v3/parser/__init__.py b/httprunner/v3/parser.py similarity index 97% rename from httprunner/v3/parser/__init__.py rename to httprunner/v3/parser.py index 1a7803a1..c1ed97a9 100644 --- a/httprunner/v3/parser/__init__.py +++ b/httprunner/v3/parser.py @@ -1,8 +1,8 @@ import re -from typing import Any, Set +from typing import Any, Set, Text from typing import Dict -from httprunner.v3.exceptions import ParamsError +from httprunner.v3 import exceptions absolute_http_url_regexp = re.compile(r"^https?://", re.I) @@ -21,7 +21,7 @@ def build_url(base_url, path): elif base_url: return "{}/{}".format(base_url.rstrip("/"), path.lstrip("/")) else: - raise ParamsError("base url missed!") + raise exceptions.ParamsError("base url missed!") def regex_findall_variables(content): @@ -58,7 +58,6 @@ def regex_findall_variables(content): return [] - def extract_variables(content: Any) -> Set: """ extract all variables in content recursively. """ diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py new file mode 100644 index 00000000..422218bf --- /dev/null +++ b/httprunner/v3/response.py @@ -0,0 +1,13 @@ +import requests + + +class ResponseObject(object): + + def __init__(self, resp_obj: requests.Response): + """ initialize with a requests.Response object + + Args: + resp_obj (instance): requests.Response instance + + """ + self.obj = resp_obj diff --git a/httprunner/v3/runner/__init__.py b/httprunner/v3/runner.py similarity index 62% rename from httprunner/v3/runner/__init__.py rename to httprunner/v3/runner.py index 61638567..7d4a12f2 100644 --- a/httprunner/v3/runner/__init__.py +++ b/httprunner/v3/runner.py @@ -2,8 +2,12 @@ from typing import List import requests +from loguru import logger + from httprunner.v3.parser import build_url from httprunner.v3.schema import TestsConfig, TestStep +from httprunner.v3.validator import Validator +from httprunner.v3.response import ResponseObject class TestCaseRunner(object): @@ -21,18 +25,35 @@ class TestCaseRunner(object): return self def run_step(self, step): - request_dict = step.request.dict() + logger.info(f"run step: {step.name}") + # prepare arguments + request_dict = step.request.dict() method = request_dict.pop("method") url_path = request_dict.pop("url") url = build_url(self.config.base_url, url_path) request_dict["json"] = request_dict.pop("req_json", {}) + logger.info(f"{method} {url}") + logger.debug(f"request kwargs(raw): {request_dict}") + + # request session = self.session or requests.Session() resp = session.request(method, url, **request_dict) - def run(self): + # validate + resp_obj = ResponseObject(resp) + validator = Validator(resp_obj) + validators = step.validation + validator.validate(validators) + + def test_start(self): + """main entrance""" for step in self.teststeps: step.variables.update(self.config.variables) self.run_step(step) + + def run(self): + """main entrance alias for test_start""" + return self.test_start() diff --git a/httprunner/v3/validator.py b/httprunner/v3/validator.py new file mode 100644 index 00000000..04940b13 --- /dev/null +++ b/httprunner/v3/validator.py @@ -0,0 +1,145 @@ +from typing import Text + +import jmespath +from loguru import logger + +from httprunner.v3.exceptions import ParamsError, ValidationFailure +from httprunner.v3.response import ResponseObject + + +def get_uniform_comparator(comparator: Text): + """ convert comparator alias to uniform name + """ + if comparator in ["eq", "equals", "==", "is"]: + return "equals" + elif comparator in ["lt", "less_than"]: + return "less_than" + elif comparator in ["le", "less_than_or_equals"]: + return "less_than_or_equals" + elif comparator in ["gt", "greater_than"]: + return "greater_than" + elif comparator in ["ge", "greater_than_or_equals"]: + return "greater_than_or_equals" + elif comparator in ["ne", "not_equals"]: + return "not_equals" + elif comparator in ["str_eq", "string_equals"]: + return "string_equals" + elif comparator in ["len_eq", "length_equals", "count_eq"]: + return "length_equals" + elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]: + return "length_greater_than" + elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", + "count_greater_than_or_equals"]: + return "length_greater_than_or_equals" + elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]: + return "length_less_than" + elif comparator in ["len_le", "count_le", "length_less_than_or_equals", + "count_less_than_or_equals"]: + return "length_less_than_or_equals" + else: + return comparator + + +def uniform_validator(validator): + """ unify validator + + Args: + validator (dict): validator maybe in two formats: + + format1: this is kept for compatiblity with the previous versions. + {"check": "status_code", "assert": "eq", "expect": 201} + {"check": "$resp_body_success", "assert": "eq", "expect": True} + format2: recommended new version, {assert: [check_item, expected_value]} + {'eq': ['status_code', 201]} + {'eq': ['$resp_body_success', True]} + + Returns + dict: validator info + + { + "check": "status_code", + "expect": 201, + "assert": "equals" + } + + """ + if not isinstance(validator, dict): + raise ParamsError(f"invalid validator: {validator}") + + if "check" in validator and "expect" in validator: + # format1 + check_item = validator["check"] + expect_value = validator["expect"] + comparator = validator.get("comparator", "eq") + + elif len(validator) == 1: + # format2 + comparator = list(validator.keys())[0] + compare_values = validator[comparator] + + if not isinstance(compare_values, list) or len(compare_values) != 2: + raise ParamsError(f"invalid validator: {validator}") + + check_item, expect_value = compare_values + + else: + raise ParamsError(f"invalid validator: {validator}") + + # uniform comparator, e.g. lt => less_than, eq => equals + assert_method = get_uniform_comparator(comparator) + + return { + "check": check_item, + "expect": expect_value, + "assert": assert_method + } + + +class AssertMethods(object): + + @staticmethod + def equals(actual_value, expect_value): + assert actual_value == expect_value + + @staticmethod + def less_than(actual_value, expect_value): + assert actual_value < expect_value + + @staticmethod + def greater_than(actual_value, expect_value): + assert actual_value > expect_value + + +class Validator(object): + + def __init__(self, resp_obj: ResponseObject): + self.resp_meta = { + "status_code": resp_obj.obj.status_code, + "headers": resp_obj.obj.headers, + "body": resp_obj.obj.json() + } + + def validate(self, validators): + + for v in validators: + u_validator = uniform_validator(v) + field = u_validator["check"] + assert_method = u_validator["assert"] + expect_value = u_validator["expect"] + actual_value = jmespath.search(field, self.resp_meta) + + msg = f"assert {field} {assert_method} {expect_value}" + + try: + assert_func = getattr(AssertMethods, assert_method) + except AttributeError: + raise ParamsError(f"Assert Method not supported: {assert_method}") + + try: + assert_func(actual_value, expect_value) + msg += " - success" + logger.info(msg) + except AssertionError: + msg += " - fail" + logger.error(msg) + raise ValidationFailure(f"assert {field}: {actual_value} {assert_method} {expect_value}") diff --git a/poetry.lock b/poetry.lock index 3dfab901..9d54bc93 100644 --- a/poetry.lock +++ b/poetry.lock @@ -173,6 +173,14 @@ MarkupSafe = ">=0.23" [package.extras] i18n = ["Babel (>=0.8)"] +[[package]] +category = "main" +description = "JSON Matching Expressions" +name = "jmespath" +optional = false +python-versions = "*" +version = "0.9.5" + [[package]] category = "main" description = "An XPath for JSON" @@ -345,7 +353,7 @@ version = "1.0.1" dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"] [metadata] -content-hash = "e1204ede1ab227bc33783b362d866c2a0b1fb8faba283216b2973e2261b0b966" +content-hash = "85e388b2b80681f40a72f5ec0a1746a5a7e834c63fbfc9e2a07058bebb6f6c29" python-versions = "^3.6" [metadata.files] @@ -470,6 +478,10 @@ jinja2 = [ {file = "Jinja2-2.10.3-py2.py3-none-any.whl", hash = "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f"}, {file = "Jinja2-2.10.3.tar.gz", hash = "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de"}, ] +jmespath = [ + {file = "jmespath-0.9.5-py2.py3-none-any.whl", hash = "sha256:695cb76fa78a10663425d5b73ddc5714eb711157e52704d69be03b1a02ba4fec"}, + {file = "jmespath-0.9.5.tar.gz", hash = "sha256:cca55c8d153173e21baa59983015ad0daf603f9cb799904ff057bfb8ff8dc2d9"}, +] jsonpath = [ {file = "jsonpath-0.82.tar.gz", hash = "sha256:46d3fd2016cd5b842283d547877a02c418a0fe9aa7a6b0ae344115a2c990fef4"}, ] diff --git a/pyproject.toml b/pyproject.toml index 3de0634c..9c47e387 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ filetype = "^1.0.5" jsonpath = "^0.82" pydantic = "^1.4" loguru = "^0.4.1" +jmespath = "^0.9.5" [tool.poetry.dev-dependencies] flask = "<1.0.0"