#29: refactor validator, now support custom defined validators

httprunner/__init__.py

@@ -1 +1 @@
-__version__ = '0.8.3'
+__version__ = '0.8.4'

httprunner/built_in.py

@@ -4,10 +4,12 @@ Built-in dependent functions used in YAML/JSON testcases.
 
 import datetime
 import random
+import re
 import string
 import time
 
 from httprunner.exception import ParamsError
+from httprunner.utils import string_type
 
 
 def gen_random_string(str_len):
@@ -33,3 +35,69 @@ def sleep(sec):
     """ sleep specified seconds
     """
     time.sleep(sec)
+
+
+""" built-in comparators
+"""
+def equals(check_value, expect_value):
+    assert check_value == expect_value
+
+def less_than(check_value, expect_value):
+    assert check_value < expect_value
+
+def less_than_or_equals(check_value, expect_value):
+    assert check_value <= expect_value
+
+def greater_than(check_value, expect_value):
+    assert check_value > expect_value
+
+def greater_than_or_equals(check_value, expect_value):
+    assert check_value >= expect_value
+
+def not_equals(check_value, expect_value):
+    assert check_value != expect_value
+
+def string_equals(check_value, expect_value):
+    assert str(check_value) == str(expect_value)
+
+def length_equals(check_value, expect_value):
+    assert isinstance(expect_value, int)
+    assert len(check_value) == expect_value
+
+def length_greater_than(check_value, expect_value):
+    assert isinstance(expect_value, int)
+    assert len(check_value) > expect_value
+
+def length_greater_than_or_equals(check_value, expect_value):
+    assert isinstance(expect_value, int)
+    assert len(check_value) >= expect_value
+
+def length_less_than(check_value, expect_value):
+    assert isinstance(expect_value, int)
+    assert len(check_value) < expect_value
+
+def length_less_than_or_equals(check_value, expect_value):
+    assert isinstance(expect_value, int)
+    assert len(check_value) <= expect_value
+
+def contains(check_value, expect_value):
+    assert isinstance(check_value, (list, tuple, dict, string_type))
+    assert expect_value in check_value
+
+def contained_by(check_value, expect_value):
+    assert isinstance(expect_value, (list, tuple, dict, string_type))
+    assert check_value in expect_value
+
+def type_match(check_value, expect_value):
+    assert isinstance(check_value, expect_value)
+
+def regex_match(check_value, expect_value):
+    assert isinstance(expect_value, string_type)
+    assert isinstance(check_value, string_type)
+    assert re.match(expect_value, check_value)
+
+def startswith(check_value, expect_value):
+    assert str(check_value).startswith(str(expect_value))
+
+def endswith(check_value, expect_value):
+    assert str(check_value).endswith(str(expect_value))

httprunner/context.py

@@ -165,6 +165,9 @@ class Context(object):
     def get_testcase_variables_mapping(self):
         return self.testcase_variables_mapping
 
+    def get_testcase_functions_mapping(self):
+        return self.testcase_functions_config
+
     def exec_content_functions(self, content):
         """ execute functions in content.
         """

httprunner/response.py

@@ -137,8 +137,13 @@ class ResponseObject(object):
             {
                 "resp_body_success": True
             }
-        @return validator info
+        @return (dict) validator info
-            check_item, check_value, expect_value, comparator
+            {
+                "check_item": check_item,
+                "check_value": check_value,
+                "expect_value": expect_value,
+                "comparator": comparator
+            }
         """
         if not isinstance(validator, dict):
             raise exception.ParamsError("invalid validator: {}".format(validator))
@@ -177,19 +182,46 @@ class ResponseObject(object):
             except exception.ParseResponseError:
                 raise exception.ParseResponseError("failed to extract check item in response!")
 
-        return check_item, check_value, expect_value, comparator
+        validator_dict = {
+            "check_item": check_item,
+            "check_value": check_value,
+            "expect_value": expect_value,
+            "comparator": comparator
+        }
+        return validator_dict
 
-    def validate(self, validators, variables_mapping):
+    def do_validation(self, validator_dict, functions_mapping):
+        """ validate with functions
+        """
+        comparator = utils.get_uniform_comparator(validator_dict["comparator"])
+
+        validate_func = functions_mapping.get(comparator)
+        if not validate_func:
+            raise exception.FunctionNotFound("comparator not found: {}".format(comparator))
+
+        check_item = validator_dict["check_item"]
+        check_value = validator_dict["check_value"]
+        expect_value = validator_dict["expect_value"]
+
+        try:
+            if check_value is None or expect_value is None:
+                assert comparator in ["is", "eq", "equals", "=="]
+
+            validate_func(validator_dict["check_value"], validator_dict["expect_value"])
+        except (AssertionError, TypeError):
+            err_msg = "\n" + "\n".join([
+                "\tcheck item name: %s;" % check_item,
+                "\tcheck item value: %s (%s);" % (check_value, type(check_value).__name__),
+                "\tcomparator: %s;" % comparator,
+                "\texpected value: %s (%s)." % (expect_value, type(expect_value).__name__)
+            ])
+            raise exception.ValidationError(err_msg)
+
+    def validate(self, validators, variables_mapping, functions_mapping):
         """ check validators with the context variable mapping.
         """
         for validator in validators:
-            check_item, check_value, expect_value, comparator = self.parse_validator(validator, variables_mapping)
+            validator_dict = self.parse_validator(validator, variables_mapping)
-
+            self.do_validation(validator_dict, functions_mapping)
-            utils.match_expected(
-                check_value,
-                expect_value,
-                comparator,
-                check_item
-            )
 
         return True

httprunner/runner.py

@@ -129,7 +129,11 @@ class Runner(object):
             self.context.bind_extracted_variables(extracted_variables_mapping)
 
             try:
-                resp_obj.validate(validators, self.context.get_testcase_variables_mapping())
+                resp_obj.validate(
+                    validators,
+                    self.context.get_testcase_variables_mapping(),
+                    self.context.get_testcase_functions_mapping()
+                )
             except (exception.ParamsError, exception.ResponseError, exception.ValidationError):
                 err_msg = u"Exception occured.\n"
                 err_msg += u"HTTP request url: {}\n".format(url)

httprunner/utils.py

@@ -117,79 +117,37 @@ def query_json(json_content, query, delimiter='.'):
 
     return json_content
 
-def match_expected(value, expected, comparator="eq", check_item=""):
+def get_uniform_comparator(comparator):
-    """ check if value matches expected value.
+    """ convert comparator alias to uniform name
-    @param value: actual value that get from response.
-    @param expected: expected result described in testcase
-    @param comparator: compare method
-    @param check_item: check item name
     """
-    try:
+    if comparator in ["eq", "equals", "=="]:
-        if value is None or expected is None:
+        return "equals"
-            assert comparator in ["is", "eq", "equals", "=="]
+    elif comparator in ["lt", "less_than"]:
-            assert value is None
+        return "less_than"
-            assert expected is None
+    elif comparator in ["le", "less_than_or_equals"]:
-
+        return "less_than_or_equals"
-        if comparator in ["eq", "equals", "=="]:
+    elif comparator in ["gt", "greater_than"]:
-            assert value == expected
+        return "greater_than"
-        elif comparator in ["lt", "less_than"]:
+    elif comparator in ["ge", "greater_than_or_equals"]:
-            assert value < expected
+        return "greater_than_or_equals"
-        elif comparator in ["le", "less_than_or_equals"]:
+    elif comparator in ["ne", "not_equals"]:
-            assert value <= expected
+        return "not_equals"
-        elif comparator in ["gt", "greater_than"]:
+    elif comparator in ["str_eq", "string_equals"]:
-            assert value > expected
+        return "string_equals"
-        elif comparator in ["ge", "greater_than_or_equals"]:
+    elif comparator in ["len_eq", "length_equals", "count_eq"]:
-            assert value >= expected
+        return "length_equals"
-        elif comparator in ["ne", "not_equals"]:
+    elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]:
-            assert value != expected
+        return "length_greater_than"
-        elif comparator in ["str_eq", "string_equals"]:
+    elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \
-            assert str(value) == str(expected)
+        "count_greater_than_or_equals"]:
-        elif comparator in ["len_eq", "length_equals", "count_eq"]:
+        return "length_greater_than_or_equals"
-            assert isinstance(expected, int)
+    elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]:
-            assert len(value) == expected
+        return "length_less_than"
-        elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]:
+    elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \
-            assert isinstance(expected, int)
+        "count_less_than_or_equals"]:
-            assert len(value) > expected
+        return "length_less_than_or_equals"
-        elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \
+    else:
-            "count_greater_than_or_equals"]:
+        return comparator
-            assert isinstance(expected, int)
-            assert len(value) >= expected
-        elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]:
-            assert isinstance(expected, int)
-            assert len(value) < expected
-        elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \
-            "count_less_than_or_equals"]:
-            assert isinstance(expected, int)
-            assert len(value) <= expected
-        elif comparator in ["contains"]:
-            assert isinstance(value, (list, tuple, dict, string_type))
-            assert expected in value
-        elif comparator in ["contained_by"]:
-            assert isinstance(expected, (list, tuple, dict, string_type))
-            assert value in expected
-        elif comparator in ["type"]:
-            assert isinstance(value, expected)
-        elif comparator in ["regex"]:
-            assert isinstance(expected, string_type)
-            assert isinstance(value, string_type)
-            assert re.match(expected, value)
-        elif comparator in ["startswith"]:
-            assert str(value).startswith(str(expected))
-        elif comparator in ["endswith"]:
-            assert str(value).endswith(str(expected))
-        else:
-            raise exception.ParamsError("comparator not supported!")
-
-        return True
-
-    except (AssertionError, TypeError):
-        err_msg = "\n".join([
-            "check item name: %s;" % check_item,
-            "check item value: %s (%s);" % (value, type(value).__name__),
-            "comparator: %s;" % comparator,
-            "expected value: %s (%s)." % (expected, type(expected).__name__)
-        ])
-        raise exception.ValidationError(err_msg)
 
 def deep_update_dict(origin_dict, override_dict):
     """ update origin dict with override dict recursively

tests/test_response.py

@@ -1,9 +1,14 @@
 import requests
-from httprunner import response, exception
+from httprunner import exception, response, utils
 from tests.base import ApiServerUnittest
 
+
 class TestResponse(ApiServerUnittest):
 
+    def setUp(self):
+        imported_module = utils.get_imported_module("httprunner.built_in")
+        self.functions_mapping = utils.filter_module(imported_module, "function")
+
     def test_parse_response_object_json(self):
         url = "http://127.0.0.1:5000/api/users"
         resp = requests.get(url)
@@ -226,6 +231,20 @@ class TestResponse(ApiServerUnittest):
         with self.assertRaises(exception.ParamsError):
             resp_obj.extract_response(extract_binds_list)
 
+    def test_do_validation(self):
+        url = "http://127.0.0.1:5000/"
+        resp = requests.get(url)
+        resp_obj = response.ResponseObject(resp)
+
+        resp_obj.do_validation(
+            {"check_item": "check_item", "check_value": 1, "expect_value": 1, "comparator": "eq"},
+            self.functions_mapping
+        )
+        resp_obj.do_validation(
+            {"check_item": "check_item", "check_value": "abc", "expect_value": "abc", "comparator": "=="},
+            self.functions_mapping
+        )
+
     def test_validate(self):
         url = "http://127.0.0.1:5000/"
         resp = requests.get(url)
@@ -241,7 +260,7 @@ class TestResponse(ApiServerUnittest):
         }
 
         with self.assertRaises(exception.ValidationError):
-            resp_obj.validate(validators, variables_mapping)
+            resp_obj.validate(validators, variables_mapping, self.functions_mapping)
 
         validators = [
             {"check": "resp_status_code", "comparator": "eq", "expect": 201},
@@ -252,7 +271,7 @@ class TestResponse(ApiServerUnittest):
             "resp_body_success": True
         }
 
-        self.assertTrue(resp_obj.validate(validators, variables_mapping))
+        self.assertTrue(resp_obj.validate(validators, variables_mapping, self.functions_mapping))
 
     def test_validate_exception(self):
         url = "http://127.0.0.1:5000/"
@@ -266,7 +285,7 @@ class TestResponse(ApiServerUnittest):
         ]
         variables_mapping = {}
         with self.assertRaises(exception.ValidationError):
-            resp_obj.validate(validators, variables_mapping)
+            resp_obj.validate(validators, variables_mapping, self.functions_mapping)
 
         # expected value missed in variables mapping
         validators = [
@@ -277,4 +296,4 @@ class TestResponse(ApiServerUnittest):
             "resp_status_code": 200
         }
         with self.assertRaises(exception.ValidationError):
-            resp_obj.validate(validators, variables_mapping)
+            resp_obj.validate(validators, variables_mapping, self.functions_mapping)

tests/test_utils.py

@@ -94,52 +94,73 @@ class TestUtils(ApiServerUnittest):
         with self.assertRaises(exception.ParseResponseError):
             utils.query_json(json_content, query)
 
-    def test_match_expected(self):
+    def test_get_uniform_comparator(self):
-        self.assertTrue(utils.match_expected(1, 1, "eq"))
+        self.assertEqual(utils.get_uniform_comparator("eq"), "equals")
-        self.assertTrue(utils.match_expected("abc", "abc", "=="))
+        self.assertEqual(utils.get_uniform_comparator("=="), "equals")
-        self.assertTrue(utils.match_expected("abc", "abc"))
+        self.assertEqual(utils.get_uniform_comparator("lt"), "less_than")
+        self.assertEqual(utils.get_uniform_comparator("le"), "less_than_or_equals")
+        self.assertEqual(utils.get_uniform_comparator("gt"), "greater_than")
+        self.assertEqual(utils.get_uniform_comparator("ge"), "greater_than_or_equals")
+        self.assertEqual(utils.get_uniform_comparator("ne"), "not_equals")
 
-        with self.assertRaises(exception.ValidationError):
+        self.assertEqual(utils.get_uniform_comparator("str_eq"), "string_equals")
-            utils.match_expected(123, "123", "eq")
+        self.assertEqual(utils.get_uniform_comparator("len_eq"), "length_equals")
-        with self.assertRaises(exception.ValidationError):
+        self.assertEqual(utils.get_uniform_comparator("count_eq"), "length_equals")
-            utils.match_expected(123, "123")
 
-        self.assertTrue(utils.match_expected(1, 2, "lt"))
+        self.assertEqual(utils.get_uniform_comparator("len_gt"), "length_greater_than")
-        self.assertTrue(utils.match_expected(1, 1, "le"))
+        self.assertEqual(utils.get_uniform_comparator("count_gt"), "length_greater_than")
-        self.assertTrue(utils.match_expected(2, 1, "gt"))
+        self.assertEqual(utils.get_uniform_comparator("count_greater_than"), "length_greater_than")
-        self.assertTrue(utils.match_expected(1, 1, "ge"))
-        self.assertTrue(utils.match_expected(123, "123", "ne"))
 
-        self.assertTrue(utils.match_expected("123", 3, "len_eq"))
+        self.assertEqual(utils.get_uniform_comparator("len_ge"), "length_greater_than_or_equals")
-        self.assertTrue(utils.match_expected("123", 2, "len_gt"))
+        self.assertEqual(utils.get_uniform_comparator("count_ge"), "length_greater_than_or_equals")
-        self.assertTrue(utils.match_expected("123", 3, "len_ge"))
+        self.assertEqual(utils.get_uniform_comparator("count_greater_than_or_equals"), "length_greater_than_or_equals")
-        self.assertTrue(utils.match_expected("123", 4, "len_lt"))
-        self.assertTrue(utils.match_expected("123", 3, "len_le"))
 
-        self.assertTrue(utils.match_expected("123abc456", "3ab", "contains"))
+        self.assertEqual(utils.get_uniform_comparator("len_lt"), "length_less_than")
-        self.assertTrue(utils.match_expected(['1', '2'], "1", "contains"))
+        self.assertEqual(utils.get_uniform_comparator("count_lt"), "length_less_than")
-        self.assertTrue(utils.match_expected({'a':1, 'b':2}, "a", "contains"))
+        self.assertEqual(utils.get_uniform_comparator("count_less_than"), "length_less_than")
-        self.assertTrue(utils.match_expected("3ab", "123abc456", "contained_by"))
 
-        self.assertTrue(utils.match_expected("123abc456", "^123\w+456$", "regex"))
+        self.assertEqual(utils.get_uniform_comparator("len_le"), "length_less_than_or_equals")
-        with self.assertRaises(exception.ValidationError):
+        self.assertEqual(utils.get_uniform_comparator("count_le"), "length_less_than_or_equals")
-            utils.match_expected("123abc456", "^12b.*456$", "regex")
+        self.assertEqual(utils.get_uniform_comparator("count_less_than_or_equals"), "length_less_than_or_equals")
 
-        with self.assertRaises(exception.ParamsError):
+    def test_validators(self):
-            utils.match_expected(1, 2, "not_supported_comparator")
+        imported_module = utils.get_imported_module("httprunner.built_in")
+        functions_mapping = utils.filter_module(imported_module, "function")
 
-        self.assertTrue(utils.match_expected("abc123", "ab", "startswith"))
+        functions_mapping["equals"](None, None)
-        self.assertTrue(utils.match_expected("123abc", 12, "startswith"))
+        functions_mapping["equals"](1, 1)
-        self.assertTrue(utils.match_expected(12345, 123, "startswith"))
+        functions_mapping["equals"]("abc", "abc")
-        self.assertTrue(utils.match_expected("abc123", 23, "endswith"))
+        with self.assertRaises(AssertionError):
-        self.assertTrue(utils.match_expected("123abc", "abc", "endswith"))
+            functions_mapping["equals"]("123", 123)
-        self.assertTrue(utils.match_expected(12345, 45, "endswith"))
 
-        self.assertTrue(utils.match_expected(None, None, "eq"))
+        functions_mapping["less_than"](1, 2)
-        with self.assertRaises(exception.ValidationError):
+        functions_mapping["less_than_or_equals"](2, 2)
-            utils.match_expected(None, 3, "len_eq")
+
-        with self.assertRaises(exception.ValidationError):
+        functions_mapping["greater_than"](2, 1)
-            utils.match_expected("abc", None, "gt")
+        functions_mapping["greater_than_or_equals"](2, 2)
+
+        functions_mapping["not_equals"](123, "123")
+
+        functions_mapping["length_equals"]("123", 3)
+        functions_mapping["length_greater_than"]("123", 2)
+        functions_mapping["length_greater_than_or_equals"]("123", 3)
+
+        functions_mapping["contains"]("123abc456", "3ab")
+        functions_mapping["contains"](['1', '2'], "1")
+        functions_mapping["contains"]({'a':1, 'b':2}, "a")
+        functions_mapping["contained_by"]("3ab", "123abc456")
+
+        functions_mapping["regex_match"]("123abc456", "^123\w+456$")
+        with self.assertRaises(AssertionError):
+            functions_mapping["regex_match"]("123abc456", "^12b.*456$")
+
+        functions_mapping["startswith"]("abc123", "ab")
+        functions_mapping["startswith"]("123abc", 12)
+        functions_mapping["startswith"](12345, 123)
+
+        functions_mapping["endswith"]("abc123", 23)
+        functions_mapping["endswith"]("123abc", "abc")
+        functions_mapping["endswith"](12345, 45)
 
     def test_deep_update_dict(self):
         origin_dict = {'a': 1, 'b': {'c': 3, 'd': 4}, 'f': 6}