change v3: add basic validation

examples/postman_echo/request_methods/hardcode_test.py

@@ -24,7 +24,8 @@ class TestCaseRequestMethodsHardcode(TestCaseRunner):
                 }
             },
             "validate": [
-                {"eq": ["status_code", 200]}
+                {"eq": ["status_code", 200]},
+                {"eq": ["headers.Server", "nginx"]}
             ]
         }),
         TestStep(**{

httprunner/v3/parser.py

@@ -1,8 +1,8 @@
 import re
-from typing import Any, Set
+from typing import Any, Set, Text
 from typing import Dict
 
-from httprunner.v3.exceptions import ParamsError
+from httprunner.v3 import exceptions
 
 absolute_http_url_regexp = re.compile(r"^https?://", re.I)
 
@@ -21,7 +21,7 @@ def build_url(base_url, path):
     elif base_url:
         return "{}/{}".format(base_url.rstrip("/"), path.lstrip("/"))
     else:
-        raise ParamsError("base url missed!")
+        raise exceptions.ParamsError("base url missed!")
 
 
 def regex_findall_variables(content):
@@ -58,7 +58,6 @@ def regex_findall_variables(content):
         return []
 
 
-
 def extract_variables(content: Any) -> Set:
     """ extract all variables in content recursively.
     """

httprunner/v3/response.py

@@ -0,0 +1,13 @@
+import requests
+
+
+class ResponseObject(object):
+
+    def __init__(self, resp_obj: requests.Response):
+        """ initialize with a requests.Response object
+
+        Args:
+            resp_obj (instance): requests.Response instance
+
+        """
+        self.obj = resp_obj

httprunner/v3/runner.py

@@ -2,8 +2,12 @@ from typing import List
 
 import requests
 
+from loguru import logger
+
 from httprunner.v3.parser import build_url
 from httprunner.v3.schema import TestsConfig, TestStep
+from httprunner.v3.validator import Validator
+from httprunner.v3.response import ResponseObject
 
 
 class TestCaseRunner(object):
@@ -21,18 +25,35 @@ class TestCaseRunner(object):
         return self
 
     def run_step(self, step):
-        request_dict = step.request.dict()
+        logger.info(f"run step: {step.name}")
 
+        # prepare arguments
+        request_dict = step.request.dict()
         method = request_dict.pop("method")
         url_path = request_dict.pop("url")
         url = build_url(self.config.base_url, url_path)
 
         request_dict["json"] = request_dict.pop("req_json", {})
 
+        logger.info(f"{method} {url}")
+        logger.debug(f"request kwargs(raw): {request_dict}")
+
+        # request
         session = self.session or requests.Session()
         resp = session.request(method, url, **request_dict)
 
-    def run(self):
+        # validate
+        resp_obj = ResponseObject(resp)
+        validator = Validator(resp_obj)
+        validators = step.validation
+        validator.validate(validators)
+
+    def test_start(self):
+        """main entrance"""
         for step in self.teststeps:
             step.variables.update(self.config.variables)
             self.run_step(step)
+
+    def run(self):
+        """main entrance alias for test_start"""
+        return self.test_start()

httprunner/v3/validator.py

@@ -0,0 +1,145 @@
+from typing import Text
+
+import jmespath
+from loguru import logger
+
+from httprunner.v3.exceptions import ParamsError, ValidationFailure
+from httprunner.v3.response import ResponseObject
+
+
+def get_uniform_comparator(comparator: Text):
+    """ convert comparator alias to uniform name
+    """
+    if comparator in ["eq", "equals", "==", "is"]:
+        return "equals"
+    elif comparator in ["lt", "less_than"]:
+        return "less_than"
+    elif comparator in ["le", "less_than_or_equals"]:
+        return "less_than_or_equals"
+    elif comparator in ["gt", "greater_than"]:
+        return "greater_than"
+    elif comparator in ["ge", "greater_than_or_equals"]:
+        return "greater_than_or_equals"
+    elif comparator in ["ne", "not_equals"]:
+        return "not_equals"
+    elif comparator in ["str_eq", "string_equals"]:
+        return "string_equals"
+    elif comparator in ["len_eq", "length_equals", "count_eq"]:
+        return "length_equals"
+    elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]:
+        return "length_greater_than"
+    elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals",
+                        "count_greater_than_or_equals"]:
+        return "length_greater_than_or_equals"
+    elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]:
+        return "length_less_than"
+    elif comparator in ["len_le", "count_le", "length_less_than_or_equals",
+                        "count_less_than_or_equals"]:
+        return "length_less_than_or_equals"
+    else:
+        return comparator
+
+
+def uniform_validator(validator):
+    """ unify validator
+
+    Args:
+        validator (dict): validator maybe in two formats:
+
+            format1: this is kept for compatiblity with the previous versions.
+                {"check": "status_code", "assert": "eq", "expect": 201}
+                {"check": "$resp_body_success", "assert": "eq", "expect": True}
+            format2: recommended new version, {assert: [check_item, expected_value]}
+                {'eq': ['status_code', 201]}
+                {'eq': ['$resp_body_success', True]}
+
+    Returns
+        dict: validator info
+
+            {
+                "check": "status_code",
+                "expect": 201,
+                "assert": "equals"
+            }
+
+    """
+    if not isinstance(validator, dict):
+        raise ParamsError(f"invalid validator: {validator}")
+
+    if "check" in validator and "expect" in validator:
+        # format1
+        check_item = validator["check"]
+        expect_value = validator["expect"]
+        comparator = validator.get("comparator", "eq")
+
+    elif len(validator) == 1:
+        # format2
+        comparator = list(validator.keys())[0]
+        compare_values = validator[comparator]
+
+        if not isinstance(compare_values, list) or len(compare_values) != 2:
+            raise ParamsError(f"invalid validator: {validator}")
+
+        check_item, expect_value = compare_values
+
+    else:
+        raise ParamsError(f"invalid validator: {validator}")
+
+    # uniform comparator, e.g. lt => less_than, eq => equals
+    assert_method = get_uniform_comparator(comparator)
+
+    return {
+        "check": check_item,
+        "expect": expect_value,
+        "assert": assert_method
+    }
+
+
+class AssertMethods(object):
+
+    @staticmethod
+    def equals(actual_value, expect_value):
+        assert actual_value == expect_value
+
+    @staticmethod
+    def less_than(actual_value, expect_value):
+        assert actual_value < expect_value
+
+    @staticmethod
+    def greater_than(actual_value, expect_value):
+        assert actual_value > expect_value
+
+
+class Validator(object):
+
+    def __init__(self, resp_obj: ResponseObject):
+        self.resp_meta = {
+            "status_code": resp_obj.obj.status_code,
+            "headers": resp_obj.obj.headers,
+            "body": resp_obj.obj.json()
+        }
+
+    def validate(self, validators):
+
+        for v in validators:
+            u_validator = uniform_validator(v)
+            field = u_validator["check"]
+            assert_method = u_validator["assert"]
+            expect_value = u_validator["expect"]
+            actual_value = jmespath.search(field, self.resp_meta)
+
+            msg = f"assert {field} {assert_method} {expect_value}"
+
+            try:
+                assert_func = getattr(AssertMethods, assert_method)
+            except AttributeError:
+                raise ParamsError(f"Assert Method not supported: {assert_method}")
+
+            try:
+                assert_func(actual_value, expect_value)
+                msg += " - success"
+                logger.info(msg)
+            except AssertionError:
+                msg += " - fail"
+                logger.error(msg)
+                raise ValidationFailure(f"assert {field}: {actual_value} {assert_method} {expect_value}")

poetry.lock

@@ -173,6 +173,14 @@ MarkupSafe = ">=0.23"
 [package.extras]
 i18n = ["Babel (>=0.8)"]
 
+[[package]]
+category = "main"
+description = "JSON Matching Expressions"
+name = "jmespath"
+optional = false
+python-versions = "*"
+version = "0.9.5"
+
 [[package]]
 category = "main"
 description = "An XPath for JSON"
@@ -345,7 +353,7 @@ version = "1.0.1"
 dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"]
 
 [metadata]
-content-hash = "e1204ede1ab227bc33783b362d866c2a0b1fb8faba283216b2973e2261b0b966"
+content-hash = "85e388b2b80681f40a72f5ec0a1746a5a7e834c63fbfc9e2a07058bebb6f6c29"
 python-versions = "^3.6"
 
 [metadata.files]
@@ -470,6 +478,10 @@ jinja2 = [
     {file = "Jinja2-2.10.3-py2.py3-none-any.whl", hash = "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f"},
     {file = "Jinja2-2.10.3.tar.gz", hash = "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de"},
 ]
+jmespath = [
+    {file = "jmespath-0.9.5-py2.py3-none-any.whl", hash = "sha256:695cb76fa78a10663425d5b73ddc5714eb711157e52704d69be03b1a02ba4fec"},
+    {file = "jmespath-0.9.5.tar.gz", hash = "sha256:cca55c8d153173e21baa59983015ad0daf603f9cb799904ff057bfb8ff8dc2d9"},
+]
 jsonpath = [
     {file = "jsonpath-0.82.tar.gz", hash = "sha256:46d3fd2016cd5b842283d547877a02c418a0fe9aa7a6b0ae344115a2c990fef4"},
 ]

pyproject.toml

@@ -37,6 +37,7 @@ filetype = "^1.0.5"
 jsonpath = "^0.82"
 pydantic = "^1.4"
 loguru = "^0.4.1"
+jmespath = "^0.9.5"
 
 [tool.poetry.dev-dependencies]
 flask = "<1.0.0"