diff --git a/httprunner/api.py b/httprunner/api.py index 4e9f27dc..0790939a 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -74,11 +74,9 @@ class HttpRunner(object): return test test_suite = unittest.TestSuite() - functions = tests_mapping.get("project_mapping", {}).get("functions", {}) - for testcase in tests_mapping["testcases"]: config = testcase.get("config", {}) - test_runner = runner.Runner(config, functions) + test_runner = runner.Runner(config) TestSequense = type('TestSequense', (unittest.TestCase,), {}) tests = testcase.get("teststeps", []) diff --git a/httprunner/cli.py b/httprunner/cli.py index 5d758e6a..888f6a97 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -8,8 +8,9 @@ def main_hrun(): from httprunner.__about__ import __description__, __version__ from httprunner.api import HttpRunner from httprunner.compat import is_py2 + from httprunner.validator import validate_json_file from httprunner.utils import (create_scaffold, get_python2_retire_msg, - prettify_json_file, validate_json_file) + prettify_json_file) parser = argparse.ArgumentParser(description=__description__) parser.add_argument( diff --git a/httprunner/context.py b/httprunner/context.py index 2f1f3700..c95e3549 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -5,18 +5,16 @@ class SessionContext(object): """ HttpRunner session, store runtime variables. Examples: - >>> functions={...} >>> variables = {"SECRET_KEY": "DebugTalk"} - >>> context = SessionContext(functions, variables) + >>> context = SessionContext(variables) Equivalent to: - >>> context = SessionContext(functions) + >>> context = SessionContext() >>> context.update_session_variables(variables) """ - def __init__(self, functions, variables=None): + def __init__(self, variables=None): self.session_variables_mapping = utils.ensure_mapping_format(variables or {}) - self.FUNCTIONS_MAPPING = functions self.init_test_variables() self.validation_results = [] @@ -63,32 +61,20 @@ class SessionContext(object): """ return parser.parse_lazy_data(content, self.test_variables_mapping) - def __eval_check_item(self, validator, resp_obj): + def __eval_validator_check(self, check_item, resp_obj): """ evaluate check item in validator. Args: - validator (dict): validator - {"check": "status_code", "comparator": "eq", "expect": 201} - {"check": "$resp_body_success", "comparator": "eq", "expect": True} - resp_obj (object): requests.Response() object + check_item: check_item should only be the following 5 formats: + 1, variable reference, e.g. $token + 2, function reference, e.g. ${is_status_code_200($status_code)} + 3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"} + 4, string joined by delimiter. e.g. "status_code", "headers.content-type" + 5, regex string, e.g. "LB[\d]*(.*)RB[\d]*" - Returns: - dict: validator info - { - "check": "status_code", - "check_value": 200, - "expect": 201, - "comparator": "eq" - } + resp_obj: response object """ - check_item = validator["check"] - # check_item should only be the following 5 formats: - # 1, variable reference, e.g. $token - # 2, function reference, e.g. ${is_status_code_200($status_code)} - # 3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"} - # 4, string joined by delimiter. e.g. "status_code", "headers.content-type" - # 5, regex string, e.g. "LB[\d]*(.*)RB[\d]*" if isinstance(check_item, (dict, list)) \ or isinstance(check_item, parser.LazyString): # format 1/2/3 @@ -97,68 +83,22 @@ class SessionContext(object): # format 4/5 check_value = resp_obj.extract_field(check_item) - validator["check_value"] = check_value + return check_value - # expect_value should only be in 2 types: - # 1, variable reference, e.g. $expect_status_code - # 2, actual value, e.g. 200 - expect_value = self.eval_content(validator["expect"]) - validator["expect"] = expect_value - validator["check_result"] = "unchecked" - return validator - - def _do_validation(self, validator_dict): - """ validate with functions + def __eval_validator_expect(self, expect_item): + """ evaluate expect item in validator. Args: - validator_dict (dict): validator dict - { - "check": "status_code", - "check_value": 200, - "expect": 201, - "comparator": "eq" - } + expect_item: expect_item should only be in 2 types: + 1, variable reference, e.g. $expect_status_code + 2, actual value, e.g. 200 """ - # TODO: move comparator uniform to init_test_suites - comparator = utils.get_uniform_comparator(validator_dict["comparator"]) - validate_func = parser.get_mapping_function(comparator, self.FUNCTIONS_MAPPING) - - check_item = validator_dict["check"] - check_value = validator_dict["check_value"] - expect_value = validator_dict["expect"] - - if (check_value is None or expect_value is None) \ - and comparator not in ["is", "eq", "equals", "=="]: - raise exceptions.ParamsError("Null value can only be compared with comparator: eq/equals/==") - - validate_msg = "validate: {} {} {}({})".format( - check_item, - comparator, - expect_value, - type(expect_value).__name__ - ) - - try: - validator_dict["check_result"] = "pass" - validate_func(check_value, expect_value) - validate_msg += "\t==> pass" - logger.log_debug(validate_msg) - except (AssertionError, TypeError): - validate_msg += "\t==> fail" - validate_msg += "\n{}({}) {} {}({})".format( - check_value, - type(check_value).__name__, - comparator, - expect_value, - type(expect_value).__name__ - ) - logger.log_error(validate_msg) - validator_dict["check_result"] = "fail" - raise exceptions.ValidationFailure(validate_msg) + expect_value = self.eval_content(expect_item) + return expect_value def validate(self, validators, resp_obj): - """ make validations + """ make validation with comparators """ self.validation_results = [] if not validators: @@ -170,19 +110,59 @@ class SessionContext(object): failures = [] for validator in validators: - # evaluate validators with context variable mapping. - evaluated_validator = self.__eval_check_item( - parser.parse_validator(validator), + # validator should be LazyFunction object + if not isinstance(validator, parser.LazyFunction): + raise exceptions.ValidationFailure( + "validator should be parsed first: {}".format(validators)) + + # evaluate validator args with context variable mapping. + validator_args = validator._args + check_item, expect_item = validator_args + check_value = self.__eval_validator_check( + check_item, resp_obj ) + expect_value = self.__eval_validator_expect(expect_item) + validator._args = [check_value, expect_value] + + comparator = validator._func.__name__ + validator_dict = { + "comparator": comparator, + "check": check_item, + "check_value": check_value, + "expect": expect_item, + "expect_value": expect_value + } + validate_msg = "\nvalidate: {} {} {}({})".format( + check_item, + comparator, + expect_value, + type(expect_value).__name__ + ) try: - self._do_validation(evaluated_validator) - except exceptions.ValidationFailure as ex: + validator.to_value(self.test_variables_mapping) + validator_dict["check_result"] = "pass" + validate_msg += "\t==> pass" + logger.log_debug(validate_msg) + except (AssertionError, TypeError): validate_pass = False - failures.append(str(ex)) + validator_dict["check_result"] = "fail" + validate_msg += "\t==> fail" + validate_msg += "\n{}({}) {} {}({})".format( + check_value, + type(check_value).__name__, + comparator, + expect_value, + type(expect_value).__name__ + ) + logger.log_error(validate_msg) + failures.append(validate_msg) - self.validation_results.append(evaluated_validator) + self.validation_results.append(validator_dict) + + # restore validator args, in case of running multiple times + validator._args = validator_args if not validate_pass: failures_string = "\n".join([failure for failure in failures]) diff --git a/httprunner/parser.py b/httprunner/parser.py index 59597c2e..cc1db6aa 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -4,7 +4,7 @@ import ast import os import re -from httprunner import exceptions, utils +from httprunner import exceptions, utils, validator from httprunner.compat import basestring, builtin_str, numeric_types, str # TODO: change variable notation from $var to {{var}} @@ -105,65 +105,6 @@ def regex_findall_functions(content): return [] -def parse_validator(validator): - """ parse validator - - Args: - validator (dict): validator maybe in two formats: - - format1: this is kept for compatiblity with the previous versions. - {"check": "status_code", "comparator": "eq", "expect": 201} - {"check": "$resp_body_success", "comparator": "eq", "expect": True} - format2: recommended new version - {'eq': ['status_code', 201]} - {'eq': ['$resp_body_success', True]} - - Returns - dict: validator info - - { - "check": "status_code", - "expect": 201, - "comparator": "eq" - } - - """ - if not isinstance(validator, dict): - raise exceptions.ParamsError("invalid validator: {}".format(validator)) - - if "check" in validator and len(validator) > 1: - # format1 - check_item = validator.get("check") - - if "expect" in validator: - expect_value = validator.get("expect") - elif "expected" in validator: - expect_value = validator.get("expected") - else: - raise exceptions.ParamsError("invalid validator: {}".format(validator)) - - comparator = validator.get("comparator", "eq") - - elif len(validator) == 1: - # format2 - comparator = list(validator.keys())[0] - compare_values = validator[comparator] - - if not isinstance(compare_values, list) or len(compare_values) != 2: - raise exceptions.ParamsError("invalid validator: {}".format(validator)) - - check_item, expect_value = compare_values - - else: - raise exceptions.ParamsError("invalid validator: {}".format(validator)) - - return { - "check": check_item, - "expect": expect_value, - "comparator": comparator - } - - def parse_parameters(parameters, variables_mapping=None, functions_mapping=None): """ parse parameters and generate cartesian product. @@ -738,12 +679,9 @@ def _extend_with_api(test_dict, api_def_dict): """ extend test with api definition, test will merge and override api definition. Args: - test_dict (dict): test block + test_dict (dict): test block, this will override api_def_dict api_def_dict (dict): api definition - Returns: - dict: extended test dict. - Examples: >>> api_def_dict = { "name": "get token 1", @@ -756,6 +694,7 @@ def _extend_with_api(test_dict, api_def_dict): "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] } >>> _extend_with_api(test_dict, api_def_dict) + >>> print(test_dict) { "name": "get token 2", "request": {...}, @@ -764,9 +703,8 @@ def _extend_with_api(test_dict, api_def_dict): } """ - # override name - api_def_name = api_def_dict.pop("name", "") - test_dict["name"] = test_dict.get("name") or api_def_name + # override api name + test_dict.setdefault("name", api_def_dict.pop("name", "")) # override variables def_variables = api_def_dict.pop("variables", []) @@ -777,16 +715,12 @@ def _extend_with_api(test_dict, api_def_dict): # merge & override validators TODO: relocate def_raw_validators = api_def_dict.pop("validate", []) - ref_raw_validators = test_dict.get("validate", []) def_validators = [ - parse_validator(validator) - for validator in def_raw_validators + validator.uniform_validator(_validator) + for _validator in def_raw_validators ] - ref_validators = [ - parse_validator(validator) - for validator in ref_raw_validators - ] - test_dict["validate"] = utils.extend_validators( + ref_validators = test_dict.pop("validate", []) + test_dict["validate"] = validator.extend_validators( def_validators, ref_validators ) @@ -798,7 +732,7 @@ def _extend_with_api(test_dict, api_def_dict): test_dict.get("extract", {}) ) - # TODO: merge & override request + # merge & override request test_dict["request"] = api_def_dict.pop("request", {}) # base_url & verify: priority api_def_dict > test_dict @@ -824,8 +758,6 @@ def _extend_with_api(test_dict, api_def_dict): # TODO: extend with other api definition items, e.g. times test_dict.update(api_def_dict) - return test_dict - def _extend_with_testcase(test_dict, testcase_def_dict): """ extend test with testcase definition @@ -923,6 +855,14 @@ def __prepare_testcase_tests(tests, config, project_mapping): if (not test_dict.get("base_url")) and config_base_url: test_dict["base_url"] = config_base_url + # unify validators' format + if "validate" in test_dict: + ref_raw_validators = test_dict.pop("validate", []) + test_dict["validate"] = [ + validator.uniform_validator(_validator) + for _validator in ref_raw_validators + ] + if "testcase_def" in test_dict: # test_dict is nested testcase @@ -954,6 +894,27 @@ def __prepare_testcase_tests(tests, config, project_mapping): check_variables_set = set(test_dict_variables.keys()) \ | set(session_variables.keys()) | {"request", "response"} + # convert validators to lazy function + validators = test_dict.pop("validate", []) + prepared_validators = [] + for _validator in validators: + function_meta = { + "func_name": _validator["comparator"], + "args": [ + _validator["check"], + _validator["expect"] + ], + "kwargs": {} + } + prepared_validators.append( + LazyFunction( + function_meta, + functions, + check_variables_set + ) + ) + test_dict["validate"] = prepared_validators + # convert variables and functions to lazy object. # raises VariableNotFound if undefined variable exists in test_dict prepared_test_dict = prepare_lazy_data( diff --git a/httprunner/runner.py b/httprunner/runner.py index 162aad10..25dfd13a 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -11,27 +11,40 @@ class Runner(object): """ Running testcases. Examples: - >>> functions={...} - >>> config = { - "name": "XXXX", - "base_url": "http://127.0.0.1", - "verify": False + >>> tests_mapping = { + "project_mapping": { + "functions": {} + }, + "testcases": [ + { + "config": { + "name": "XXXX", + "base_url": "http://127.0.0.1", + "verify": False + }, + "teststeps": [ + { + "name": "test description", + "variables": [], # optional + "request": { + "url": "http://127.0.0.1:5000/api/users/1000", + "method": "GET" + } + } + ] + } + ] } - >>> runner = Runner(config, functions) - >>> test_dict = { - "name": "test description", - "variables": [], # optional - "request": { - "url": "http://127.0.0.1:5000/api/users/1000", - "method": "GET" - } - } - >>> runner.run_test(test_dict) + >>> parsed_tests_mapping = parser.parse_tests(tests_mapping) + >>> parsed_testcase = parsed_tests_mapping["testcases"][0] + + >>> test_runner = runner.Runner(parsed_testcase["config"]) + >>> test_runner.run_test(parsed_testcase["teststeps"][0]) """ - def __init__(self, config, functions, http_client_session=None): + def __init__(self, config, http_client_session=None): """ run testcase or testsuite. Args: @@ -50,7 +63,6 @@ class Runner(object): base_url = config.get("base_url") self.verify = config.get("verify", True) self.output = config.get("output", []) - self.functions = functions self.validation_results = [] # testcase setup hooks @@ -59,7 +71,7 @@ class Runner(object): self.testcase_teardown_hooks = config.get("teardown_hooks", []) self.http_client_session = http_client_session or HttpSession(base_url) - self.session_context = SessionContext(self.functions) + self.session_context = SessionContext() if testcase_setup_hooks: self.do_hook_actions(testcase_setup_hooks, "setup") @@ -289,7 +301,7 @@ class Runner(object): config = testcase_dict.get("config", {}) # each teststeps in one testcase (YAML/JSON) share the same session. - test_runner = Runner(config, self.functions, self.http_client_session) + test_runner = Runner(config, self.http_client_session) tests = testcase_dict.get("teststeps", []) diff --git a/httprunner/templates/locustfile_template b/httprunner/templates/locustfile_template index c7582549..f8d81b61 100644 --- a/httprunner/templates/locustfile_template +++ b/httprunner/templates/locustfile_template @@ -15,7 +15,7 @@ logging.getLogger('locust.runners').setLevel(logging.INFO) class WebPageTasks(TaskSet): def on_start(self): - self.test_runner = Runner(self.locust.config, self.locust.functions, self.client) + self.test_runner = Runner(self.locust.config, self.client) @task def test_any(self): @@ -38,7 +38,6 @@ class WebPageUser(HttpLocust): file_path = "$TESTCASE_FILE" locust_tests = prepare_locust_tests(file_path) - functions = locust_tests["functions"] tests = locust_tests["tests"] config = {} diff --git a/httprunner/utils.py b/httprunner/utils.py index 12ec7717..7747715c 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -120,39 +120,6 @@ def query_json(json_content, query, delimiter='.'): return json_content -def get_uniform_comparator(comparator): - """ convert comparator alias to uniform name - """ - if comparator in ["eq", "equals", "==", "is"]: - return "equals" - elif comparator in ["lt", "less_than"]: - return "less_than" - elif comparator in ["le", "less_than_or_equals"]: - return "less_than_or_equals" - elif comparator in ["gt", "greater_than"]: - return "greater_than" - elif comparator in ["ge", "greater_than_or_equals"]: - return "greater_than_or_equals" - elif comparator in ["ne", "not_equals"]: - return "not_equals" - elif comparator in ["str_eq", "string_equals"]: - return "string_equals" - elif comparator in ["len_eq", "length_equals", "count_eq"]: - return "length_equals" - elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]: - return "length_greater_than" - elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \ - "count_greater_than_or_equals"]: - return "length_greater_than_or_equals" - elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]: - return "length_less_than" - elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \ - "count_less_than_or_equals"]: - return "length_less_than_or_equals" - else: - return comparator - - def deep_update_dict(origin_dict, override_dict): """ update origin dict with override dict recursively e.g. origin_dict = {'a': 1, 'b': {'c': 2, 'd': 4}} @@ -323,78 +290,6 @@ def ensure_mapping_format(variables): raise exceptions.ParamsError("variables format error!") -def _convert_validators_to_mapping(validators): - """ convert validators list to mapping. - - Args: - validators (list): validators in list - - Returns: - dict: validators mapping, use (check, comparator) as key. - - Examples: - >>> validators = [ - {"check": "v1", "expect": 201, "comparator": "eq"}, - {"check": {"b": 1}, "expect": 200, "comparator": "eq"} - ] - >>> _convert_validators_to_mapping(validators) - { - ("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"}, - ('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"} - } - - """ - validators_mapping = {} - - for validator in validators: - if not isinstance(validator["check"], collections.Hashable): - check = json.dumps(validator["check"]) - else: - check = validator["check"] - - key = (check, validator["comparator"]) - validators_mapping[key] = validator - - return validators_mapping - - -def extend_validators(raw_validators, override_validators): - """ extend raw_validators with override_validators. - override_validators will merge and override raw_validators. - - Args: - raw_validators (dict): - override_validators (dict): - - Returns: - list: extended validators - - Examples: - >>> raw_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] - >>> override_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] - >>> extend_validators(raw_validators, override_validators) - [ - {"check": "v1", "expect": 201, "comparator": "eq"}, - {"check": "s2", "expect": 16, "comparator": "len_eq"}, - {"check": "s3", "expect": 12, "comparator": "len_eq"} - ] - - """ - - if not raw_validators: - return override_validators - - elif not override_validators: - return raw_validators - - else: - def_validators_mapping = _convert_validators_to_mapping(raw_validators) - ref_validators_mapping = _convert_validators_to_mapping(override_validators) - - def_validators_mapping.update(ref_validators_mapping) - return list(def_validators_mapping.values()) - - def extend_variables(raw_variables, override_variables): """ extend raw_variables with override_variables. override_variables will merge and override raw_variables. @@ -581,25 +476,6 @@ def gen_cartesian_product(*args): return product_list -def validate_json_file(file_list): - """ validate JSON testcase format - """ - for json_file in set(file_list): - if not json_file.endswith(".json"): - logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file)) - continue - - logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN") - - with io.open(json_file) as stream: - try: - json.load(stream) - except ValueError as e: - raise SystemExit(e) - - print("OK") - - def prettify_json_file(file_list): """ prettify JSON testcase format """ diff --git a/httprunner/validator.py b/httprunner/validator.py index 6e52c332..60c15cd1 100644 --- a/httprunner/validator.py +++ b/httprunner/validator.py @@ -1,7 +1,12 @@ # encoding: utf-8 +import collections +import io +import json import os import types +from httprunner import exceptions, logger + """ validate data format TODO: refactor with JSON schema validate @@ -129,6 +134,170 @@ def is_testcase_path(path): return True +############################################################################### +## testcase validator utils +############################################################################### + +def get_uniform_comparator(comparator): + """ convert comparator alias to uniform name + """ + if comparator in ["eq", "equals", "==", "is"]: + return "equals" + elif comparator in ["lt", "less_than"]: + return "less_than" + elif comparator in ["le", "less_than_or_equals"]: + return "less_than_or_equals" + elif comparator in ["gt", "greater_than"]: + return "greater_than" + elif comparator in ["ge", "greater_than_or_equals"]: + return "greater_than_or_equals" + elif comparator in ["ne", "not_equals"]: + return "not_equals" + elif comparator in ["str_eq", "string_equals"]: + return "string_equals" + elif comparator in ["len_eq", "length_equals", "count_eq"]: + return "length_equals" + elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]: + return "length_greater_than" + elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \ + "count_greater_than_or_equals"]: + return "length_greater_than_or_equals" + elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]: + return "length_less_than" + elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \ + "count_less_than_or_equals"]: + return "length_less_than_or_equals" + else: + return comparator + + +def uniform_validator(validator): + """ unify validator + + Args: + validator (dict): validator maybe in two formats: + + format1: this is kept for compatiblity with the previous versions. + {"check": "status_code", "comparator": "eq", "expect": 201} + {"check": "$resp_body_success", "comparator": "eq", "expect": True} + format2: recommended new version, {comparator: [check_item, expected_value]} + {'eq': ['status_code', 201]} + {'eq': ['$resp_body_success', True]} + + Returns + dict: validator info + + { + "check": "status_code", + "expect": 201, + "comparator": "equals" + } + + """ + if not isinstance(validator, dict): + raise exceptions.ParamsError("invalid validator: {}".format(validator)) + + if "check" in validator and "expect" in validator: + # format1 + check_item = validator["check"] + expect_value = validator["expect"] + comparator = validator.get("comparator", "eq") + + elif len(validator) == 1: + # format2 + comparator = list(validator.keys())[0] + compare_values = validator[comparator] + + if not isinstance(compare_values, list) or len(compare_values) != 2: + raise exceptions.ParamsError("invalid validator: {}".format(validator)) + + check_item, expect_value = compare_values + + else: + raise exceptions.ParamsError("invalid validator: {}".format(validator)) + + # uniform comparator, e.g. lt => less_than, eq => equals + comparator = get_uniform_comparator(comparator) + + return { + "check": check_item, + "expect": expect_value, + "comparator": comparator + } + + +def _convert_validators_to_mapping(validators): + """ convert validators list to mapping. + + Args: + validators (list): validators in list + + Returns: + dict: validators mapping, use (check, comparator) as key. + + Examples: + >>> validators = [ + {"check": "v1", "expect": 201, "comparator": "eq"}, + {"check": {"b": 1}, "expect": 200, "comparator": "eq"} + ] + >>> _convert_validators_to_mapping(validators) + { + ("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"}, + ('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"} + } + + """ + validators_mapping = {} + + for validator in validators: + if not isinstance(validator["check"], collections.Hashable): + check = json.dumps(validator["check"]) + else: + check = validator["check"] + + key = (check, validator["comparator"]) + validators_mapping[key] = validator + + return validators_mapping + + +def extend_validators(raw_validators, override_validators): + """ extend raw_validators with override_validators. + override_validators will merge and override raw_validators. + + Args: + raw_validators (dict): + override_validators (dict): + + Returns: + list: extended validators + + Examples: + >>> raw_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] + >>> override_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] + >>> extend_validators(raw_validators, override_validators) + [ + {"check": "v1", "expect": 201, "comparator": "eq"}, + {"check": "s2", "expect": 16, "comparator": "len_eq"}, + {"check": "s3", "expect": 12, "comparator": "len_eq"} + ] + + """ + + if not raw_validators: + return override_validators + + elif not override_validators: + return raw_validators + + else: + def_validators_mapping = _convert_validators_to_mapping(raw_validators) + ref_validators_mapping = _convert_validators_to_mapping(override_validators) + + def_validators_mapping.update(ref_validators_mapping) + return list(def_validators_mapping.values()) + + ############################################################################### ## validate varibles and functions ############################################################################### @@ -157,3 +326,22 @@ def is_variable(tup): return False return True + + +def validate_json_file(file_list): + """ validate JSON testcase format + """ + for json_file in set(file_list): + if not json_file.endswith(".json"): + logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file)) + continue + + logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN") + + with io.open(json_file) as stream: + try: + json.load(stream) + except ValueError as e: + raise SystemExit(e) + + print("OK") diff --git a/tests/httpbin/api/302_redirect.yml b/tests/httpbin/api/302_redirect.yml index 8b24637f..959cc227 100644 --- a/tests/httpbin/api/302_redirect.yml +++ b/tests/httpbin/api/302_redirect.yml @@ -6,5 +6,6 @@ request: url: https://debugtalk.com status_code: 302 method: GET + verify: False validate: - eq: ["status_code", 200] diff --git a/tests/test_client.py b/tests/test_client.py index 9be921b5..ae593016 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -76,7 +76,7 @@ class TestHttpClient(ApiServerUnittest): "a": "1", "b": "2" } - resp = self.api_client.get(url, cookies=cookies, headers=self.headers) + resp = self.api_client.get(url, cookies=cookies, headers=self.headers, verify=False) raw_request = resp.history[0].request self.assertEqual(raw_request._cookies["a"], "1") self.assertEqual(raw_request._cookies["b"], "2") diff --git a/tests/test_context.py b/tests/test_context.py index b1ad8f61..7d0a5ca7 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,8 +1,7 @@ import os import time -import requests -from httprunner import context, exceptions, loader, parser, response, utils +from httprunner import context, exceptions, loader, parser, runner from tests.base import ApiServerUnittest, gen_md5, gen_random_string @@ -11,16 +10,10 @@ class TestContext(ApiServerUnittest): def setUp(self): loader.load_project_tests(os.path.join(os.getcwd(), "tests")) project_mapping = loader.project_mapping - self.functions = project_mapping["functions"] self.context = context.SessionContext( - functions=self.functions, variables={"SECRET_KEY": "DebugTalk"} ) - def test_init_context_functions(self): - context_functions = self.context.FUNCTIONS_MAPPING - self.assertIn("gen_md5", context_functions) - def test_init_test_variables_initialize(self): self.assertEqual( self.context.test_variables_mapping, @@ -57,13 +50,6 @@ class TestContext(ApiServerUnittest): "debugtalk" ) - def test_eval_content_functions(self): - content = parser.prepare_lazy_data("${sleep_N_secs(1)}", self.functions) - start_time = time.time() - self.context.eval_content(content) - elapsed_time = time.time() - start_time - self.assertGreater(elapsed_time, 1) - def test_eval_content_variables(self): variables = { "SECRET_KEY": "DebugTalk" @@ -124,82 +110,80 @@ class TestContext(ApiServerUnittest): ) self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk") - def test_do_validation(self): - self.context._do_validation( - {"check": "check", "check_value": 1, "expect": 1, "comparator": "eq"} - ) - self.context._do_validation( - {"check": "check", "check_value": "abc", "expect": "abc", "comparator": "=="} - ) - self.context._do_validation( - {"check": "status_code", "check_value": "201", "expect": 3, "comparator": "sum_status_code"} - ) - def test_validate(self): - url = "http://127.0.0.1:5000/" - resp = requests.get(url) - resp_obj = response.ResponseObject(resp) - - validators = [ - {"eq": ["$resp_status_code", 201]}, - {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, - {"check": "$resp_body_success", "comparator": "eq", "expect": True} - ] - validators = parser.prepare_lazy_data(validators, {}, {"resp_status_code", "resp_body_success"}) - variables = { - "resp_status_code": 200, - "resp_body_success": True - } - - self.context.init_test_variables(variables) - - with self.assertRaises(exceptions.ValidationFailure): - self.context.validate(validators, resp_obj) - - validators = [ - {"eq": ["$resp_status_code", 201]}, - {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, - {"check": "$resp_body_success", "comparator": "eq", "expect": True}, - {"check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False} + testcases = [ + { + "config": { + 'name': "test validation" + }, + "teststeps": [ + { + "name": "test validation", + "request": { + "url": "http://127.0.0.1:5000/", + "method": "GET", + }, + "variables": { + "resp_status_code": 200, + "resp_body_success": True + }, + "validate": [ + {"eq": ["$resp_status_code", 200]}, + {"check": "$resp_status_code", "comparator": "eq", "expect": 200}, + {"check": "$resp_body_success", "expect": True}, + {"check": "${is_status_code_200($resp_status_code)}", "expect": True} + ] + } + ] + } ] from tests.debugtalk import is_status_code_200 - functions = { - "is_status_code_200": is_status_code_200 + tests_mapping = { + "project_mapping": { + "functions": { + "is_status_code_200": is_status_code_200 + } + }, + "testcases": testcases } - validators = parser.prepare_lazy_data( - validators, functions, {"resp_status_code", "resp_body_success"}) - variables = [ - {"resp_status_code": 201}, - {"resp_body_success": True} - ] - self.context.init_test_variables(variables) - self.context.validate(validators, resp_obj) - - self.context.validate([], resp_obj) - self.assertEqual(self.context.validation_results, []) + parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_tests_mapping["testcases"][0] + test_runner = runner.Runner(parsed_testcase["config"]) + teststep = parsed_testcase["teststeps"][0] + test_runner.run_test(teststep) def test_validate_exception(self): - url = "http://127.0.0.1:5000/" - resp = requests.get(url) - resp_obj = response.ResponseObject(resp) - - # expected value missed in validators - validators = [ - {"eq": ["$resp_status_code", 201]}, - {"check": "$resp_status_code", "comparator": "eq", "expect": 201} + testcases = [ + { + "config": { + 'name': "test validation" + }, + "teststeps": [ + { + "name": "test validation", + "request": { + "url": "http://127.0.0.1:5000/", + "method": "GET", + }, + "variables": { + "resp_status_code": 200, + "resp_body_success": True + }, + "validate": [ + {"eq": ["$resp_status_code", 201]}, + {"check": "$resp_status_code", "expect": 201}, + {"check": "$resp_body_success", "comparator": "eq", "expect": True} + ] + } + ] + } ] - validators = parser.prepare_lazy_data(validators, {}, {"resp_status_code"}) - variables = [] - self.context.init_test_variables(variables) - - with self.assertRaises(exceptions.VariableNotFound): - self.context.validate(validators, resp_obj) - - # expected value missed in variables mapping - variables = [ - {"resp_status_code": 200} - ] - self.context.init_test_variables(variables) - + tests_mapping = { + "testcases": testcases + } + parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_tests_mapping["testcases"][0] + test_runner = runner.Runner(parsed_testcase["config"]) + teststep = parsed_testcase["teststeps"][0] with self.assertRaises(exceptions.ValidationFailure): - self.context.validate(validators, resp_obj) + test_runner.run_test(teststep) diff --git a/tests/test_parser.py b/tests/test_parser.py index cf996648..c0317811 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -104,19 +104,6 @@ class TestParserBasic(unittest.TestCase): {'args': ["$request", '12 3'], 'kwargs': {}} ) - def test_parse_validator(self): - validator = {"check": "status_code", "comparator": "eq", "expect": 201} - self.assertEqual( - parser.parse_validator(validator), - {"check": "status_code", "comparator": "eq", "expect": 201} - ) - - validator = {'eq': ['status_code', 201]} - self.assertEqual( - parser.parse_validator(validator), - {"check": "status_code", "comparator": "eq", "expect": 201} - ) - def test_extract_variables(self): prepared_content = parser.prepare_lazy_data("123$a", {}, {"a"}) self.assertEqual( @@ -1076,15 +1063,15 @@ class TestParser(unittest.TestCase): 'json': {'sign': '${get_sign($device_sn, $os_platform, $app_version)}'} }, 'validate': [ - {'eq': ['status_code', 201]}, - {'len_eq': ['content.token', 32]} + {"check": "status_code", "comparator": "equals", "expect": 201}, + {"check": "content.token", "comparator": "length_equals", "expect": 32} ] } - extended_block = parser._extend_with_api(test_block, api_def_dict) - self.assertEqual(extended_block["base_url"], "https://debugtalk.com") - self.assertEqual(extended_block["name"], "override block") - self.assertEqual({'var': 123}, extended_block["variables"]) - self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, extended_block["validate"]) - self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, extended_block["validate"]) - self.assertEqual(extended_block["times"], 3) + parser._extend_with_api(test_block, api_def_dict) + self.assertEqual(test_block["base_url"], "https://debugtalk.com") + self.assertEqual(test_block["name"], "override block") + self.assertEqual({'var': 123}, test_block["variables"]) + self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'equals'}, test_block["validate"]) + self.assertIn({'check': 'content.token', 'comparator': 'length_equals', 'expect': 32}, test_block["validate"]) + self.assertEqual(test_block["times"], 3) diff --git a/tests/test_runner.py b/tests/test_runner.py index a952406f..22fabe46 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,8 +1,7 @@ import os import time -from httprunner import exceptions, loader, parser, runner -from httprunner.utils import deep_update_dict +from httprunner import loader, parser, runner from tests.api_server import HTTPBIN_SERVER from tests.base import ApiServerUnittest @@ -19,7 +18,7 @@ class TestRunner(ApiServerUnittest): "base_url": "http://127.0.0.1", "verify": False } - self.test_runner = runner.Runner(config, self.debugtalk_functions) + self.test_runner = runner.Runner(config) self.reset_all() def reset_all(self): @@ -36,214 +35,253 @@ class TestRunner(ApiServerUnittest): ] for testcase_file_path in testcase_file_path_list: - testcases = loader.load_file(testcase_file_path) - testcases = parser.prepare_lazy_data( - testcases, - self.debugtalk_functions, - {"expect_status_code", "token_len", "token", "success"} - ) - config_dict = {} - test_runner = runner.Runner(config_dict, self.debugtalk_functions) - - test = testcases[0]["test"] - test_runner.run_test(test) - - test = testcases[1]["test"] - test_runner.run_test(test) - - test = testcases[2]["test"] - test_runner.run_test(test) - - def test_run_single_testcase_fail(self): - test = { - "name": "get token", - "request": { - "url": "http://127.0.0.1:5000/api/get-token", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3", - "device_sn": "HZfFBh6tU59EdXJ", - "os_platform": "ios", - "app_version": "2.8.6" - }, - "json": { - "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" - } - }, - "validate": [ - {"check": "status_code", "expect": 205}, - {"check": "content.token", "comparator": "len_eq", "expect": 19} - ] - } - - with self.assertRaises(exceptions.ValidationFailure): - self.test_runner.run_test(test) + tests_mapping = loader.load_tests(testcase_file_path) + parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_tests_mapping["testcases"][0] + test_runner = runner.Runner(parsed_testcase["config"]) + test_runner.run_test(parsed_testcase["teststeps"][0]) + test_runner.run_test(parsed_testcase["teststeps"][1]) + test_runner.run_test(parsed_testcase["teststeps"][2]) def test_run_testcase_with_hooks(self): start_time = time.time() - config_dict = { - "name": "basic test with httpbin", - "base_url": HTTPBIN_SERVER, - "setup_hooks": [ - "${sleep_N_secs(0.5)}", - "${hook_print(setup)}" - ], - "teardown_hooks": [ - "${sleep_N_secs(1)}", - "${hook_print(teardown)}" - ] + testcases = [ + { + "config": { + "name": "basic test with httpbin", + "base_url": HTTPBIN_SERVER, + "setup_hooks": [ + "${sleep_N_secs(0.5)}", + "${hook_print(setup)}" + ], + "teardown_hooks": [ + "${sleep_N_secs(1)}", + "${hook_print(teardown)}" + ] + }, + "teststeps": [ + { + "name": "get token", + "request": { + "url": "http://127.0.0.1:5000/api/get-token", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3", + "device_sn": "HZfFBh6tU59EdXJ", + "os_platform": "ios", + "app_version": "2.8.6" + }, + "json": { + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" + } + }, + "validate": [ + {"check": "status_code", "expect": 200} + ] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions + }, + "testcases": testcases } - prepared_config_dict = parser.prepare_lazy_data(config_dict, self.debugtalk_functions) - test_runner = runner.Runner(prepared_config_dict, self.debugtalk_functions) + parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_tests_mapping["testcases"][0] + test_runner = runner.Runner(parsed_testcase["config"]) end_time = time.time() # check if testcase setup hook executed self.assertGreater(end_time - start_time, 0.5) start_time = time.time() - test = { - "name": "get token", - "request": { - "url": "http://127.0.0.1:5000/api/get-token", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3", - "device_sn": "HZfFBh6tU59EdXJ", - "os_platform": "ios", - "app_version": "2.8.6" - }, - "json": { - "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" - } - }, - "validate": [ - {"check": "status_code", "expect": 200} - ] - } - test_runner.run_test(test) - test_runner.run_test(test) + test_runner.run_test(parsed_testcase["teststeps"][0]) end_time = time.time() # testcase teardown hook has not been executed now self.assertLess(end_time - start_time, 1) def test_run_testcase_with_hooks_assignment(self): - config_dict = { - "name": "basic test with httpbin", - "base_url": HTTPBIN_SERVER - } - test = { - "name": "modify request headers", - "base_url": HTTPBIN_SERVER, - "request": { - "url": "/anything", - "method": "POST", - "headers": { - "user_agent": "iOS/10.3", - "os_platform": "ios" + testcases = [ + { + "config": { + "name": "basic test with httpbin", + "base_url": HTTPBIN_SERVER }, - "data": "a=1&b=2" + "teststeps": [ + { + "name": "modify request headers", + "base_url": HTTPBIN_SERVER, + "request": { + "url": "/anything", + "method": "POST", + "headers": { + "user_agent": "iOS/10.3", + "os_platform": "ios" + }, + "data": "a=1&b=2" + }, + "setup_hooks": [ + {"total": "${sum_two(1, 5)}"} + ], + "validate": [ + {"check": "status_code", "expect": 200} + ] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "setup_hooks": [ - {"total": "${sum_two(1, 5)}"} - ], - "validate": [ - {"check": "status_code", "expect": 200} - ] + "testcases": testcases } - parsed_test = parser.prepare_lazy_data(test, self.debugtalk_functions) - test_runner = runner.Runner(config_dict, self.debugtalk_functions) - test_runner.run_test(parsed_test) + parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_tests_mapping["testcases"][0] + test_runner = runner.Runner(parsed_testcase["config"]) + test_runner.run_test(parsed_testcase["teststeps"][0]) test_variables_mapping = test_runner.session_context.test_variables_mapping self.assertEqual(test_variables_mapping["total"], 6) self.assertEqual(test_variables_mapping["request"]["data"], "a=1&b=2") def test_run_testcase_with_hooks_modify_request(self): - config_dict = { - "name": "basic test with httpbin", - "base_url": HTTPBIN_SERVER - } - test = { - "name": "modify request headers", - "base_url": HTTPBIN_SERVER, - "request": { - "url": "/anything", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3" + testcases = [ + { + "config": { + "name": "basic test with httpbin", + "base_url": HTTPBIN_SERVER }, - "json": { - "os_platform": "ios", - "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" - } + "teststeps": [ + { + "name": "modify request headers", + "base_url": HTTPBIN_SERVER, + "request": { + "url": "/anything", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3" + }, + "json": { + "os_platform": "ios", + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" + } + }, + "setup_hooks": [ + "${modify_request_json($request, android)}" + ], + "validate": [ + {"check": "status_code", "expect": 200}, + {"check": "content.json.os_platform", "expect": "android"} + ] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "setup_hooks": [ - "${modify_request_json($request, android)}" - ], - "validate": [ - {"check": "status_code", "expect": 200}, - {"check": "content.json.os_platform", "expect": "android"} - ] + "testcases": testcases } - test_runner = runner.Runner(config_dict, self.debugtalk_functions) - parsed_test = parser.prepare_lazy_data(test, self.debugtalk_functions, {"request"}) - test_runner.run_test(parsed_test) + parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_tests_mapping["testcases"][0] + test_runner = runner.Runner(parsed_testcase["config"]) + test_runner.run_test(parsed_testcase["teststeps"][0]) def test_run_testcase_with_teardown_hooks_success(self): - test = { - "name": "get token", - "request": { - "url": "http://127.0.0.1:5000/api/get-token", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3", - "device_sn": "HZfFBh6tU59EdXJ", - "os_platform": "ios", - "app_version": "2.8.6" + testcases = [ + { + "config": { + "name": "basic test with httpbin" }, - "json": { - "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" - } + "teststeps": [ + { + "name": "get token", + "request": { + "url": "http://127.0.0.1:5000/api/get-token", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3", + "device_sn": "HZfFBh6tU59EdXJ", + "os_platform": "ios", + "app_version": "2.8.6" + }, + "json": { + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" + } + }, + "validate": [ + {"check": "status_code", "expect": 200} + ], + "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "validate": [ - {"check": "status_code", "expect": 200} - ], - "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] + "testcases": testcases } + parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_tests_mapping["testcases"][0] + test_runner = runner.Runner(parsed_testcase["config"]) + start_time = time.time() - self.test_runner.run_test(test) + test_runner.run_test(parsed_testcase["teststeps"][0]) end_time = time.time() # check if teardown function executed self.assertLess(end_time - start_time, 0.5) def test_run_testcase_with_teardown_hooks_fail(self): - test = { - "name": "get token", - "request": { - "url": "http://127.0.0.1:5000/api/get-token2", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3", - "device_sn": "HZfFBh6tU59EdXJ", - "os_platform": "ios", - "app_version": "2.8.6" + testcases = [ + { + "config": { + "name": "basic test with httpbin" }, - "json": { - "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" - } + "teststeps": [ + { + "name": "get token", + "request": { + "url": "http://127.0.0.1:5000/api/get-token2", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3", + "device_sn": "HZfFBh6tU59EdXJ", + "os_platform": "ios", + "app_version": "2.8.6" + }, + "json": { + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" + } + }, + "validate": [ + {"check": "status_code", "expect": 404} + ], + "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "validate": [ - {"check": "status_code", "expect": 404} - ], - "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] + "testcases": testcases } - prepared_test = parser.prepare_lazy_data(test, self.debugtalk_functions, {"response"}) + parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_tests_mapping["testcases"][0] + test_runner = runner.Runner(parsed_testcase["config"]) + start_time = time.time() - self.test_runner.run_test(prepared_test) + test_runner.run_test(parsed_testcase["teststeps"][0]) end_time = time.time() # check if teardown function executed self.assertGreater(end_time - start_time, 2) @@ -251,34 +289,51 @@ class TestRunner(ApiServerUnittest): def test_bugfix_type_match(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/bugfix_type_match.yml') - testcases = loader.load_file(testcase_file_path) - - test = testcases[1]["test"] - self.test_runner.run_test(test) + tests_mapping = loader.load_tests(testcase_file_path) + parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_tests_mapping["testcases"][0] + test_runner = runner.Runner(parsed_testcase["config"]) + test_runner.run_test(parsed_testcase["teststeps"][0]) def test_run_validate_elapsed(self): - test = { - "name": "get token", - "request": { - "url": "http://127.0.0.1:5000/api/get-token", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3", - "device_sn": "HZfFBh6tU59EdXJ", - "os_platform": "ios", - "app_version": "2.8.6" - }, - "json": { - "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" - } + testcases = [ + { + "config": {}, + "teststeps": [ + { + "name": "get token", + "request": { + "url": "http://127.0.0.1:5000/api/get-token", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3", + "device_sn": "HZfFBh6tU59EdXJ", + "os_platform": "ios", + "app_version": "2.8.6" + }, + "json": { + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" + } + }, + "validate": [ + {"check": "status_code", "expect": 200}, + {"check": "elapsed.seconds", "comparator": "lt", "expect": 1}, + {"check": "elapsed.days", "comparator": "eq", "expect": 0}, + {"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000}, + {"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1} + ] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "validate": [ - {"check": "status_code", "expect": 200}, - {"check": "elapsed.seconds", "comparator": "lt", "expect": 1}, - {"check": "elapsed.days", "comparator": "eq", "expect": 0}, - {"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000}, - {"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1} - ] + "testcases": testcases } - self.test_runner.run_test(test) + parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_tests_mapping["testcases"][0] + test_runner = runner.Runner(parsed_testcase["config"]) + test_runner.run_test(parsed_testcase["teststeps"][0]) \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index 354a906a..9a804b2a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -61,35 +61,6 @@ class TestUtils(ApiServerUnittest): result = utils.query_json(json_content, query) self.assertEqual(result, "L") - def test_get_uniform_comparator(self): - self.assertEqual(utils.get_uniform_comparator("eq"), "equals") - self.assertEqual(utils.get_uniform_comparator("=="), "equals") - self.assertEqual(utils.get_uniform_comparator("lt"), "less_than") - self.assertEqual(utils.get_uniform_comparator("le"), "less_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("gt"), "greater_than") - self.assertEqual(utils.get_uniform_comparator("ge"), "greater_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("ne"), "not_equals") - - self.assertEqual(utils.get_uniform_comparator("str_eq"), "string_equals") - self.assertEqual(utils.get_uniform_comparator("len_eq"), "length_equals") - self.assertEqual(utils.get_uniform_comparator("count_eq"), "length_equals") - - self.assertEqual(utils.get_uniform_comparator("len_gt"), "length_greater_than") - self.assertEqual(utils.get_uniform_comparator("count_gt"), "length_greater_than") - self.assertEqual(utils.get_uniform_comparator("count_greater_than"), "length_greater_than") - - self.assertEqual(utils.get_uniform_comparator("len_ge"), "length_greater_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("count_ge"), "length_greater_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("count_greater_than_or_equals"), "length_greater_than_or_equals") - - self.assertEqual(utils.get_uniform_comparator("len_lt"), "length_less_than") - self.assertEqual(utils.get_uniform_comparator("count_lt"), "length_less_than") - self.assertEqual(utils.get_uniform_comparator("count_less_than"), "length_less_than") - - self.assertEqual(utils.get_uniform_comparator("len_le"), "length_less_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("count_le"), "length_less_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("count_less_than_or_equals"), "length_less_than_or_equals") - def current_validators(self): from httprunner import built_in functions_mapping = loader.load_module_functions(built_in) @@ -205,61 +176,6 @@ class TestUtils(ApiServerUnittest): self.assertIsInstance(ordered_dict, dict) self.assertIn("a", ordered_dict) - def test_extend_validators(self): - def_validators = [ - {'eq': ['v1', 200]}, - {"check": "s2", "expect": 16, "comparator": "len_eq"} - ] - current_validators = [ - {"check": "v1", "expect": 201}, - {'len_eq': ['s3', 12]} - ] - def_validators = [ - parser.parse_validator(validator) - for validator in def_validators - ] - ref_validators = [ - parser.parse_validator(validator) - for validator in current_validators - ] - - extended_validators = utils.extend_validators(def_validators, ref_validators) - self.assertIn( - {"check": "v1", "expect": 201, "comparator": "eq"}, - extended_validators - ) - self.assertIn( - {"check": "s2", "expect": 16, "comparator": "len_eq"}, - extended_validators - ) - self.assertIn( - {"check": "s3", "expect": 12, "comparator": "len_eq"}, - extended_validators - ) - - def test_extend_validators_with_dict(self): - def_validators = [ - {'eq': ["a", {"v": 1}]}, - {'eq': [{"b": 1}, 200]} - ] - current_validators = [ - {'len_eq': ['s3', 12]}, - {'eq': [{"b": 1}, 201]} - ] - def_validators = [ - parser.parse_validator(validator) - for validator in def_validators - ] - ref_validators = [ - parser.parse_validator(validator) - for validator in current_validators - ] - - extended_validators = utils.extend_validators(def_validators, ref_validators) - self.assertEqual(len(extended_validators), 3) - self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'eq'}, extended_validators) - self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'eq'}, extended_validators) - def test_extend_variables(self): raw_variables = [{"var1": "val1"}, {"var2": "val2"}] override_variables = [{"var1": "val111"}, {"var3": "val3"}] diff --git a/tests/test_validator.py b/tests/test_validator.py index 64e97b2d..63a8cc5a 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -74,3 +74,101 @@ class TestValidator(unittest.TestCase): func = lambda x: x + 1 self.assertTrue(validator.is_function(func)) self.assertTrue(validator.is_function(validator.is_testcase)) + + def test_get_uniform_comparator(self): + self.assertEqual(validator.get_uniform_comparator("eq"), "equals") + self.assertEqual(validator.get_uniform_comparator("=="), "equals") + self.assertEqual(validator.get_uniform_comparator("lt"), "less_than") + self.assertEqual(validator.get_uniform_comparator("le"), "less_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("gt"), "greater_than") + self.assertEqual(validator.get_uniform_comparator("ge"), "greater_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("ne"), "not_equals") + + self.assertEqual(validator.get_uniform_comparator("str_eq"), "string_equals") + self.assertEqual(validator.get_uniform_comparator("len_eq"), "length_equals") + self.assertEqual(validator.get_uniform_comparator("count_eq"), "length_equals") + + self.assertEqual(validator.get_uniform_comparator("len_gt"), "length_greater_than") + self.assertEqual(validator.get_uniform_comparator("count_gt"), "length_greater_than") + self.assertEqual(validator.get_uniform_comparator("count_greater_than"), "length_greater_than") + + self.assertEqual(validator.get_uniform_comparator("len_ge"), "length_greater_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("count_ge"), "length_greater_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("count_greater_than_or_equals"), "length_greater_than_or_equals") + + self.assertEqual(validator.get_uniform_comparator("len_lt"), "length_less_than") + self.assertEqual(validator.get_uniform_comparator("count_lt"), "length_less_than") + self.assertEqual(validator.get_uniform_comparator("count_less_than"), "length_less_than") + + self.assertEqual(validator.get_uniform_comparator("len_le"), "length_less_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("count_le"), "length_less_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("count_less_than_or_equals"), "length_less_than_or_equals") + + def test_parse_validator(self): + _validator = {"check": "status_code", "comparator": "eq", "expect": 201} + self.assertEqual( + validator.uniform_validator(_validator), + {"check": "status_code", "comparator": "equals", "expect": 201} + ) + + _validator = {'eq': ['status_code', 201]} + self.assertEqual( + validator.uniform_validator(_validator), + {"check": "status_code", "comparator": "equals", "expect": 201} + ) + + + def test_extend_validators(self): + def_validators = [ + {'eq': ['v1', 200]}, + {"check": "s2", "expect": 16, "comparator": "len_eq"} + ] + current_validators = [ + {"check": "v1", "expect": 201}, + {'len_eq': ['s3', 12]} + ] + def_validators = [ + validator.uniform_validator(_validator) + for _validator in def_validators + ] + ref_validators = [ + validator.uniform_validator(_validator) + for _validator in current_validators + ] + + extended_validators = validator.extend_validators(def_validators, ref_validators) + self.assertIn( + {"check": "v1", "expect": 201, "comparator": "equals"}, + extended_validators + ) + self.assertIn( + {"check": "s2", "expect": 16, "comparator": "length_equals"}, + extended_validators + ) + self.assertIn( + {"check": "s3", "expect": 12, "comparator": "length_equals"}, + extended_validators + ) + + def test_extend_validators_with_dict(self): + def_validators = [ + {'eq': ["a", {"v": 1}]}, + {'eq': [{"b": 1}, 200]} + ] + current_validators = [ + {'len_eq': ['s3', 12]}, + {'eq': [{"b": 1}, 201]} + ] + def_validators = [ + validator.uniform_validator(_validator) + for _validator in def_validators + ] + ref_validators = [ + validator.uniform_validator(_validator) + for _validator in current_validators + ] + + extended_validators = validator.extend_validators(def_validators, ref_validators) + self.assertEqual(len(extended_validators), 3) + self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'equals'}, extended_validators) + self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'equals'}, extended_validators)