diff --git a/HISTORY.md b/HISTORY.md index 590f10a2..926e2234 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,11 @@ # Release History +## 2.1.0 (2019-03-19) + +**Features** + +- implement json dump Python objects when save tests + ## 2.0.6 (2019-03-18) **Features** diff --git a/httprunner/__about__.py b/httprunner/__about__.py index 5f7b3a5e..01d4dc4f 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '2.0.6' +__version__ = '2.1.0' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'Apache-2.0' diff --git a/httprunner/api.py b/httprunner/api.py index 798293c2..8dc2b7d6 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -36,11 +36,11 @@ class HttpRunner(object): if log_file: logger.setup_logger(log_level, log_file) - def _add_tests(self, tests_mapping): + def _add_tests(self, testcases): """ initialize testcase with Runner() and add to test suite. Args: - tests_mapping (dict): project info and testcases list. + testcases (list): testcases list. Returns: unittest.TestSuite() @@ -60,18 +60,23 @@ class HttpRunner(object): if "config" in test_dict: # run nested testcase test.__doc__ = test_dict["config"].get("name") + variables = test_dict["config"].get("variables", {}) else: # run api test test.__doc__ = test_dict.get("name") + variables = test_dict.get("variables", {}) + + if isinstance(test.__doc__, parser.LazyString): + parsed_variables = parser.parse_variables_mapping(variables, ignore=True) + test.__doc__ = parser.parse_lazy_data( + test.__doc__, parsed_variables) return test test_suite = unittest.TestSuite() - functions = tests_mapping.get("project_mapping", {}).get("functions", {}) - - for testcase in tests_mapping["testcases"]: + for testcase in testcases: config = testcase.get("config", {}) - test_runner = runner.Runner(config, functions) + test_runner = runner.Runner(config) TestSequense = type('TestSequense', (unittest.TestCase,), {}) tests = testcase.get("teststeps", []) @@ -157,19 +162,20 @@ class HttpRunner(object): def run_tests(self, tests_mapping): """ run testcase/testsuite data """ + project_mapping = tests_mapping.get("project_mapping", {}) if self.save_tests: - utils.dump_tests(tests_mapping, "loaded") + utils.dump_logs(tests_mapping, project_mapping, "loaded") # parse tests self.exception_stage = "parse tests" - parsed_tests_mapping = parser.parse_tests(tests_mapping) + parsed_testcases = parser.parse_tests(tests_mapping) if self.save_tests: - utils.dump_tests(parsed_tests_mapping, "parsed") + utils.dump_logs(parsed_testcases, project_mapping, "parsed") # add tests to test suite self.exception_stage = "add tests to test suite" - test_suite = self._add_tests(parsed_tests_mapping) + test_suite = self._add_tests(parsed_testcases) # run test suite self.exception_stage = "run test suite" @@ -184,7 +190,7 @@ class HttpRunner(object): report.stringify_summary(self._summary) if self.save_tests: - utils.dump_summary(self._summary, tests_mapping["project_mapping"]) + utils.dump_logs(self._summary, project_mapping, "summary") report_path = report.render_html_report( self._summary, @@ -275,27 +281,22 @@ def prepare_locust_tests(path): path (str): testcase file path. Returns: - dict: locust tests data + list: locust tests data - { - "functions": {}, - "tests": [] - } + [ + testcase1_dict, + testcase2_dict + ] """ tests_mapping = loader.load_tests(path) - parsed_tests_mapping = parser.parse_tests(tests_mapping) + testcases = parser.parse_tests(tests_mapping) - functions = parsed_tests_mapping.get("project_mapping", {}).get("functions", {}) + locust_tests = [] - tests = [] - - for testcase in parsed_tests_mapping["testcases"]: + for testcase in testcases: testcase_weight = testcase.get("config", {}).pop("weight", 1) for _ in range(testcase_weight): - tests.append(testcase) + locust_tests.append(testcase) - return { - "functions": functions, - "tests": tests - } + return locust_tests diff --git a/httprunner/cli.py b/httprunner/cli.py index 5d758e6a..888f6a97 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -8,8 +8,9 @@ def main_hrun(): from httprunner.__about__ import __description__, __version__ from httprunner.api import HttpRunner from httprunner.compat import is_py2 + from httprunner.validator import validate_json_file from httprunner.utils import (create_scaffold, get_python2_retire_msg, - prettify_json_file, validate_json_file) + prettify_json_file) parser = argparse.ArgumentParser(description=__description__) parser.add_argument( diff --git a/httprunner/client.py b/httprunner/client.py index 279c775c..60018e86 100644 --- a/httprunner/client.py +++ b/httprunner/client.py @@ -5,7 +5,7 @@ import time import requests import urllib3 from httprunner import logger -from httprunner.utils import build_url, lower_dict_keys, omit_long_data +from httprunner.utils import lower_dict_keys, omit_long_data from requests import Request, Response from requests.exceptions import (InvalidSchema, InvalidURL, MissingSchema, RequestException) @@ -28,15 +28,10 @@ class HttpSession(requests.Session): display statistics. This is a slightly extended version of `python-request `_'s - :py:class:`requests.Session` class and mostly this class works exactly the same. However - the methods for making requests (get, post, delete, put, head, options, patch, request) - can now take a *url* argument that's only the path part of the URL, in which case the host - part of the URL will be prepended with the HttpSession.base_url which is normally inherited - from a HttpRunner class' host property. + :py:class:`requests.Session` class and mostly this class works exactly the same. """ - def __init__(self, base_url=None, *args, **kwargs): + def __init__(self, *args, **kwargs): super(HttpSession, self).__init__(*args, **kwargs) - self.base_url = base_url if base_url else "" self.init_meta_data() def init_meta_data(self): @@ -180,9 +175,6 @@ class HttpSession(requests.Session): kwargs.setdefault("timeout", 120) self.meta_data["data"][0]["request"].update(kwargs) - # prepend url with hostname unless it's already an absolute URL - url = build_url(self.base_url, url) - start_timestamp = time.time() response = self._send_request_safe_mode(method, url, **kwargs) response_time_ms = round((time.time() - start_timestamp) * 1000, 2) diff --git a/httprunner/context.py b/httprunner/context.py index 7d402fbf..980f44a9 100644 --- a/httprunner/context.py +++ b/httprunner/context.py @@ -5,18 +5,17 @@ class SessionContext(object): """ HttpRunner session, store runtime variables. Examples: - >>> functions={...} >>> variables = {"SECRET_KEY": "DebugTalk"} - >>> context = SessionContext(functions, variables) + >>> context = SessionContext(variables) Equivalent to: - >>> context = SessionContext(functions) + >>> context = SessionContext() >>> context.update_session_variables(variables) """ - def __init__(self, functions, variables=None): - self.session_variables_mapping = utils.ensure_mapping_format(variables or {}) - self.FUNCTIONS_MAPPING = functions + def __init__(self, variables=None): + variables_mapping = utils.ensure_mapping_format(variables or {}) + self.session_variables_mapping = parser.parse_variables_mapping(variables_mapping) self.init_test_variables() self.validation_results = [] @@ -36,16 +35,14 @@ class SessionContext(object): """ variables_mapping = variables_mapping or {} variables_mapping = utils.ensure_mapping_format(variables_mapping) + variables_mapping.update(self.session_variables_mapping) + parsed_variables_mapping = parser.parse_variables_mapping(variables_mapping) self.test_variables_mapping = {} # priority: extracted variable > teststep variable - self.test_variables_mapping.update(variables_mapping) + self.test_variables_mapping.update(parsed_variables_mapping) self.test_variables_mapping.update(self.session_variables_mapping) - for variable_name, variable_value in variables_mapping.items(): - variable_value = self.eval_content(variable_value) - self.update_test_variables(variable_name, variable_value) - def update_test_variables(self, variable_name, variable_value): """ update test variables, these variables are only valid in the current test. """ @@ -63,110 +60,46 @@ class SessionContext(object): """ evaluate content recursively, take effect on each variable and function in content. content may be in any data structure, include dict, list, tuple, number, string, etc. """ - return parser.parse_data( - content, - self.test_variables_mapping, - self.FUNCTIONS_MAPPING - ) + return parser.parse_lazy_data(content, self.test_variables_mapping) - def __eval_check_item(self, validator, resp_obj): + def __eval_validator_check(self, check_item, resp_obj): """ evaluate check item in validator. Args: - validator (dict): validator - {"check": "status_code", "comparator": "eq", "expect": 201} - {"check": "$resp_body_success", "comparator": "eq", "expect": True} - resp_obj (object): requests.Response() object + check_item: check_item should only be the following 5 formats: + 1, variable reference, e.g. $token + 2, function reference, e.g. ${is_status_code_200($status_code)} + 3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"} + 4, string joined by delimiter. e.g. "status_code", "headers.content-type" + 5, regex string, e.g. "LB[\d]*(.*)RB[\d]*" - Returns: - dict: validator info - { - "check": "status_code", - "check_value": 200, - "expect": 201, - "comparator": "eq" - } + resp_obj: response object """ - check_item = validator["check"] - # check_item should only be the following 5 formats: - # 1, variable reference, e.g. $token - # 2, function reference, e.g. ${is_status_code_200($status_code)} - # 3, dict or list, maybe containing variable/function reference, e.g. {"var": "$abc"} - # 4, string joined by delimiter. e.g. "status_code", "headers.content-type" - # 5, regex string, e.g. "LB[\d]*(.*)RB[\d]*" - if isinstance(check_item, (dict, list)) \ - or parser.extract_variables(check_item) \ - or parser.extract_functions(check_item): + or isinstance(check_item, parser.LazyString): # format 1/2/3 check_value = self.eval_content(check_item) else: # format 4/5 check_value = resp_obj.extract_field(check_item) - validator["check_value"] = check_value + return check_value - # expect_value should only be in 2 types: - # 1, variable reference, e.g. $expect_status_code - # 2, actual value, e.g. 200 - expect_value = self.eval_content(validator["expect"]) - validator["expect"] = expect_value - validator["check_result"] = "unchecked" - return validator - - def _do_validation(self, validator_dict): - """ validate with functions + def __eval_validator_expect(self, expect_item): + """ evaluate expect item in validator. Args: - validator_dict (dict): validator dict - { - "check": "status_code", - "check_value": 200, - "expect": 201, - "comparator": "eq" - } + expect_item: expect_item should only be in 2 types: + 1, variable reference, e.g. $expect_status_code + 2, actual value, e.g. 200 """ - # TODO: move comparator uniform to init_test_suites - comparator = utils.get_uniform_comparator(validator_dict["comparator"]) - validate_func = parser.get_mapping_function(comparator, self.FUNCTIONS_MAPPING) - - check_item = validator_dict["check"] - check_value = validator_dict["check_value"] - expect_value = validator_dict["expect"] - - if (check_value is None or expect_value is None) \ - and comparator not in ["is", "eq", "equals", "=="]: - raise exceptions.ParamsError("Null value can only be compared with comparator: eq/equals/==") - - validate_msg = "validate: {} {} {}({})".format( - check_item, - comparator, - expect_value, - type(expect_value).__name__ - ) - - try: - validator_dict["check_result"] = "pass" - validate_func(check_value, expect_value) - validate_msg += "\t==> pass" - logger.log_debug(validate_msg) - except (AssertionError, TypeError): - validate_msg += "\t==> fail" - validate_msg += "\n{}({}) {} {}({})".format( - check_value, - type(check_value).__name__, - comparator, - expect_value, - type(expect_value).__name__ - ) - logger.log_error(validate_msg) - validator_dict["check_result"] = "fail" - raise exceptions.ValidationFailure(validate_msg) + expect_value = self.eval_content(expect_item) + return expect_value def validate(self, validators, resp_obj): - """ make validations + """ make validation with comparators """ self.validation_results = [] if not validators: @@ -178,19 +111,59 @@ class SessionContext(object): failures = [] for validator in validators: - # evaluate validators with context variable mapping. - evaluated_validator = self.__eval_check_item( - parser.parse_validator(validator), + # validator should be LazyFunction object + if not isinstance(validator, parser.LazyFunction): + raise exceptions.ValidationFailure( + "validator should be parsed first: {}".format(validators)) + + # evaluate validator args with context variable mapping. + validator_args = validator.get_args() + check_item, expect_item = validator_args + check_value = self.__eval_validator_check( + check_item, resp_obj ) + expect_value = self.__eval_validator_expect(expect_item) + validator.update_args([check_value, expect_value]) + + comparator = validator.func_name + validator_dict = { + "comparator": comparator, + "check": check_item, + "check_value": check_value, + "expect": expect_item, + "expect_value": expect_value + } + validate_msg = "\nvalidate: {} {} {}({})".format( + check_item, + comparator, + expect_value, + type(expect_value).__name__ + ) try: - self._do_validation(evaluated_validator) - except exceptions.ValidationFailure as ex: + validator.to_value(self.test_variables_mapping) + validator_dict["check_result"] = "pass" + validate_msg += "\t==> pass" + logger.log_debug(validate_msg) + except (AssertionError, TypeError): validate_pass = False - failures.append(str(ex)) + validator_dict["check_result"] = "fail" + validate_msg += "\t==> fail" + validate_msg += "\n{}({}) {} {}({})".format( + check_value, + type(check_value).__name__, + comparator, + expect_value, + type(expect_value).__name__ + ) + logger.log_error(validate_msg) + failures.append(validate_msg) - self.validation_results.append(evaluated_validator) + self.validation_results.append(validator_dict) + + # restore validator args, in case of running multiple times + validator.update_args(validator_args) if not validate_pass: failures_string = "\n".join([failure for failure in failures]) diff --git a/httprunner/parser.py b/httprunner/parser.py index 4b0c5f75..b9c3a36c 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -4,12 +4,14 @@ import ast import os import re -from httprunner import exceptions, utils +from httprunner import exceptions, utils, validator from httprunner.compat import basestring, builtin_str, numeric_types, str -variable_regexp = r"\$([\w_]+)" -function_regexp = r"\$\{([\w_]+\([\$\w\.\-/_ =,]*\))\}" -function_regexp_compile = re.compile(r"^([\w_]+)\(([\$\w\.\-/_ =,]*)\)$") +# TODO: change variable notation from $var to {{var}} +# $var_1 +variable_regex_compile = re.compile(r"\$(\w+)") +# ${func1($var_1, $var_3)} +function_regex_compile = re.compile(r"\$\{(\w+)\(([\$\w\.\-/\s=,]*)\)\}") def parse_string_value(str_value): @@ -28,7 +30,21 @@ def parse_string_value(str_value): return str_value -def extract_variables(content): +def is_variable_exist(content): + if not isinstance(content, basestring): + return False + + return True if variable_regex_compile.search(content) else False + + +def is_function_exist(content): + if not isinstance(content, basestring): + return False + + return True if function_regex_compile.search(content) else False + + +def regex_findall_variables(content): """ extract all variable names from content, which is in format $variable Args: @@ -38,27 +54,26 @@ def extract_variables(content): list: variables list extracted from string content Examples: - >>> extract_variables("$variable") + >>> regex_findall_variables("$variable") ["variable"] - >>> extract_variables("/blog/$postid") + >>> regex_findall_variables("/blog/$postid") ["postid"] - >>> extract_variables("/$var1/$var2") + >>> regex_findall_variables("/$var1/$var2") ["var1", "var2"] - >>> extract_variables("abc") + >>> regex_findall_variables("abc") [] """ - # TODO: change variable notation from $var to {{var}} try: - return re.findall(variable_regexp, content) + return variable_regex_compile.findall(content) except TypeError: return [] -def extract_functions(content): +def regex_findall_functions(content): """ extract all functions from string content, which are in format ${fun()} Args: @@ -68,200 +83,28 @@ def extract_functions(content): list: functions list extracted from string content Examples: - >>> extract_functions("${func(5)}") + >>> regex_findall_functions("${func(5)}") ["func(5)"] - >>> extract_functions("${func(a=1, b=2)}") + >>> regex_findall_functions("${func(a=1, b=2)}") ["func(a=1, b=2)"] - >>> extract_functions("/api/1000?_t=${get_timestamp()}") + >>> regex_findall_functions("/api/1000?_t=${get_timestamp()}") ["get_timestamp()"] - >>> extract_functions("/api/${add(1, 2)}") + >>> regex_findall_functions("/api/${add(1, 2)}") ["add(1, 2)"] - >>> extract_functions("/api/${add(1, 2)}?_t=${get_timestamp()}") + >>> regex_findall_functions("/api/${add(1, 2)}?_t=${get_timestamp()}") ["add(1, 2)", "get_timestamp()"] """ try: - return re.findall(function_regexp, content) + return function_regex_compile.findall(content) except TypeError: return [] -def parse_function(content): - """ parse function name and args from string content. - - Args: - content (str): string content - - Returns: - dict: function meta dict - - { - "func_name": "xxx", - "args": [], - "kwargs": {} - } - - Examples: - >>> parse_function("func()") - {'func_name': 'func', 'args': [], 'kwargs': {}} - - >>> parse_function("func(5)") - {'func_name': 'func', 'args': [5], 'kwargs': {}} - - >>> parse_function("func(1, 2)") - {'func_name': 'func', 'args': [1, 2], 'kwargs': {}} - - >>> parse_function("func(a=1, b=2)") - {'func_name': 'func', 'args': [], 'kwargs': {'a': 1, 'b': 2}} - - >>> parse_function("func(1, 2, a=3, b=4)") - {'func_name': 'func', 'args': [1, 2], 'kwargs': {'a':3, 'b':4}} - - """ - matched = function_regexp_compile.match(content) - if not matched: - raise exceptions.FunctionNotFound("{} not found!".format(content)) - - function_meta = { - "func_name": matched.group(1), - "args": [], - "kwargs": {} - } - - args_str = matched.group(2).strip() - if args_str == "": - return function_meta - - args_list = args_str.split(',') - for arg in args_list: - arg = arg.strip() - if '=' in arg: - key, value = arg.split('=') - function_meta["kwargs"][key.strip()] = parse_string_value(value.strip()) - else: - function_meta["args"].append(parse_string_value(arg)) - - return function_meta - - -def parse_validator(validator): - """ parse validator - - Args: - validator (dict): validator maybe in two formats: - - format1: this is kept for compatiblity with the previous versions. - {"check": "status_code", "comparator": "eq", "expect": 201} - {"check": "$resp_body_success", "comparator": "eq", "expect": True} - format2: recommended new version - {'eq': ['status_code', 201]} - {'eq': ['$resp_body_success', True]} - - Returns - dict: validator info - - { - "check": "status_code", - "expect": 201, - "comparator": "eq" - } - - """ - if not isinstance(validator, dict): - raise exceptions.ParamsError("invalid validator: {}".format(validator)) - - if "check" in validator and len(validator) > 1: - # format1 - check_item = validator.get("check") - - if "expect" in validator: - expect_value = validator.get("expect") - elif "expected" in validator: - expect_value = validator.get("expected") - else: - raise exceptions.ParamsError("invalid validator: {}".format(validator)) - - comparator = validator.get("comparator", "eq") - - elif len(validator) == 1: - # format2 - comparator = list(validator.keys())[0] - compare_values = validator[comparator] - - if not isinstance(compare_values, list) or len(compare_values) != 2: - raise exceptions.ParamsError("invalid validator: {}".format(validator)) - - check_item, expect_value = compare_values - - else: - raise exceptions.ParamsError("invalid validator: {}".format(validator)) - - return { - "check": check_item, - "expect": expect_value, - "comparator": comparator - } - - -def substitute_variables(content, variables_mapping): - """ substitute variables in content with variables_mapping - - Args: - content (str/dict/list/numeric/bool/type): content to be substituted. - variables_mapping (dict): variables mapping. - - Returns: - substituted content. - - Examples: - >>> content = { - 'request': { - 'url': '/api/users/$uid', - 'headers': {'token': '$token'} - } - } - >>> variables_mapping = {"$uid": 1000} - >>> substitute_variables(content, variables_mapping) - { - 'request': { - 'url': '/api/users/1000', - 'headers': {'token': '$token'} - } - } - - """ - if isinstance(content, (list, set, tuple)): - return [ - substitute_variables(item, variables_mapping) - for item in content - ] - - if isinstance(content, dict): - substituted_data = {} - for key, value in content.items(): - eval_key = substitute_variables(key, variables_mapping) - eval_value = substitute_variables(value, variables_mapping) - substituted_data[eval_key] = eval_value - - return substituted_data - - if isinstance(content, basestring): - # content is in string format here - for var, value in variables_mapping.items(): - if content == var: - # content is a variable - content = value - else: - if not isinstance(value, str): - value = builtin_str(value) - content = content.replace(var, value) - - return content - def parse_parameters(parameters, variables_mapping=None, functions_mapping=None): """ parse parameters and generate cartesian product. @@ -314,7 +157,14 @@ def parse_parameters(parameters, variables_mapping=None, functions_mapping=None) parameter_content_list.append(parameter_content_dict) else: # (2) & (3) - parsed_parameter_content = parse_data(parameter_content, variables_mapping, functions_mapping) + parsed_variables_mapping = parse_variables_mapping( + variables_mapping + ) + parsed_parameter_content = eval_lazy_data( + parameter_content, + parsed_variables_mapping, + functions_mapping + ) if not isinstance(parsed_parameter_content, list): raise exceptions.ParamsError("parameters syntax error!") @@ -390,6 +240,13 @@ def get_mapping_function(function_name, functions_mapping): if function_name in functions_mapping: return functions_mapping[function_name] + elif function_name in ["parameterize", "P"]: + from httprunner import loader + return loader.load_csv_file + + elif function_name in ["environ", "ENV"]: + return utils.get_os_environ + try: # check if HttpRunner builtin functions from httprunner import loader @@ -409,221 +266,452 @@ def get_mapping_function(function_name, functions_mapping): raise exceptions.FunctionNotFound("{} is not found.".format(function_name)) -def parse_string_functions(content, variables_mapping, functions_mapping): - """ parse string content with functions mapping. +def parse_function_params(params): + """ parse function params to args and kwargs. Args: - content (str): string content to be parsed. - variables_mapping (dict): variables mapping. - functions_mapping (dict): functions mapping. + params (str): function param in string Returns: - str: parsed string content. + dict: function meta dict - Examples: - >>> content = "abc${add_one(3)}def" - >>> functions_mapping = {"add_one": lambda x: x + 1} - >>> parse_string_functions(content, functions_mapping) - "abc4def" - - """ - functions_list = extract_functions(content) - for func_content in functions_list: - function_meta = parse_function(func_content) - func_name = function_meta["func_name"] - - args = function_meta.get("args", []) - kwargs = function_meta.get("kwargs", {}) - args = parse_data(args, variables_mapping, functions_mapping) - kwargs = parse_data(kwargs, variables_mapping, functions_mapping) - - if func_name in ["parameterize", "P"]: - if len(args) != 1 or kwargs: - raise exceptions.ParamsError("P() should only pass in one argument!") - from httprunner import loader - eval_value = loader.load_csv_file(args[0]) - elif func_name in ["environ", "ENV"]: - if len(args) != 1 or kwargs: - raise exceptions.ParamsError("ENV() should only pass in one argument!") - eval_value = utils.get_os_environ(args[0]) - else: - func = get_mapping_function(func_name, functions_mapping) - eval_value = func(*args, **kwargs) - - func_content = "${" + func_content + "}" - if func_content == content: - # content is a function, e.g. "${add_one(3)}" - content = eval_value - else: - # content contains one or many functions, e.g. "abc${add_one(3)}def" - content = content.replace( - func_content, - str(eval_value), 1 - ) - - return content - - -def parse_string_variables(content, variables_mapping, functions_mapping): - """ parse string content with variables mapping. - - Args: - content (str): string content to be parsed. - variables_mapping (dict): variables mapping. - - Returns: - str: parsed string content. - - Examples: - >>> content = "/api/users/$uid" - >>> variables_mapping = {"$uid": 1000} - >>> parse_string_variables(content, variables_mapping, {}) - "/api/users/1000" - - """ - variables_list = extract_variables(content) - for variable_name in variables_list: - variable_value = get_mapping_variable(variable_name, variables_mapping) - - if variable_name == "request" and isinstance(variable_value, dict) \ - and "url" in variable_value and "method" in variable_value: - # call setup_hooks action with $request - for key, value in variable_value.items(): - variable_value[key] = parse_data( - value, - variables_mapping, - functions_mapping - ) - parsed_variable_value = variable_value - elif "${}".format(variable_name) == variable_value: - # variable_name = "token" - # variables_mapping = {"token": "$token"} - parsed_variable_value = variable_value - else: - parsed_variable_value = parse_data( - variable_value, - variables_mapping, - functions_mapping, - raise_if_variable_not_found=False - ) - variables_mapping[variable_name] = parsed_variable_value - # TODO: replace variable label from $var to {{var}} - if "${}".format(variable_name) == content: - # content is a variable - content = parsed_variable_value - else: - # content contains one or several variables - if not isinstance(parsed_variable_value, str): - parsed_variable_value = builtin_str(parsed_variable_value) - - content = content.replace( - "${}".format(variable_name), - parsed_variable_value, 1 - ) - - return content - - -def parse_data(content, variables_mapping=None, functions_mapping=None, raise_if_variable_not_found=True): - """ parse content with variables mapping - - Args: - content (str/dict/list/numeric/bool/type): content to be parsed - variables_mapping (dict): variables mapping. - functions_mapping (dict): functions mapping. - raise_if_variable_not_found (bool): if set False, exception will not raise when VariableNotFound occurred. - - Returns: - parsed content. - - Examples: - >>> content = { - 'request': { - 'url': '/api/users/$uid', - 'headers': {'token': '$token'} - } - } - >>> variables_mapping = {"uid": 1000, "token": "abcdef"} - >>> parse_data(content, variables_mapping) { - 'request': { - 'url': '/api/users/1000', - 'headers': {'token': 'abcdef'} - } + "args": [], + "kwargs": {} } + Examples: + >>> parse_function_params("") + {'args': [], 'kwargs': {}} + + >>> parse_function_params("5") + {'args': [5], 'kwargs': {}} + + >>> parse_function_params("1, 2") + {'args': [1, 2], 'kwargs': {}} + + >>> parse_function_params("a=1, b=2") + {'args': [], 'kwargs': {'a': 1, 'b': 2}} + + >>> parse_function_params("1, 2, a=3, b=4") + {'args': [1, 2], 'kwargs': {'a':3, 'b':4}} + + """ + function_meta = { + "args": [], + "kwargs": {} + } + + params_str = params.strip() + if params_str == "": + return function_meta + + args_list = params_str.split(',') + for arg in args_list: + arg = arg.strip() + if '=' in arg: + key, value = arg.split('=') + function_meta["kwargs"][key.strip()] = parse_string_value(value.strip()) + else: + function_meta["args"].append(parse_string_value(arg)) + + return function_meta + + +class LazyFunction(object): + """ call function lazily. + """ + def __init__(self, function_meta, functions_mapping=None, check_variables_set=None): + """ init LazyFunction object with function_meta + + Args: + function_meta (dict): function name, args and kwargs. + { + "func_name": "func", + "args": [1, 2] + "kwargs": {"a": 3, "b": 4} + } + + """ + self.functions_mapping = functions_mapping or {} + self.check_variables_set = check_variables_set or set() + self.cache_key = None + self.__parse(function_meta) + + def __parse(self, function_meta): + """ init func as lazy functon instance + + Args: + function_meta (dict): function meta including name, args and kwargs + """ + self._func = get_mapping_function( + function_meta["func_name"], + self.functions_mapping + ) + self.func_name = self._func.__name__ + self._args = prepare_lazy_data( + function_meta.get("args", []), + self.functions_mapping, + self.check_variables_set + ) + self._kwargs = prepare_lazy_data( + function_meta.get("kwargs", {}), + self.functions_mapping, + self.check_variables_set + ) + + if self.func_name == "load_csv_file": + if len(self._args) != 1 or self._kwargs: + raise exceptions.ParamsError("P() should only pass in one argument!") + self._args = [self._args[0]] + elif self.func_name == "get_os_environ": + if len(self._args) != 1 or self._kwargs: + raise exceptions.ParamsError("ENV() should only pass in one argument!") + self._args = [self._args[0]] + + def get_args(self): + return self._args + + def update_args(self, args): + self._args = args + + def __repr__(self): + args_string = "" + + if self._args: + str_args = [str(arg) for arg in self._args] + args_string += ", ".join(str_args) + + if self._kwargs: + args_string += ", " + str_kwargs = [ + "{}={}".format(key, str(value)) + for key, value in self._kwargs.items() + ] + args_string += ", ".join(str_kwargs) + + return "LazyFunction({}({}))".format(self.func_name, args_string) + + def __prepare_cache_key(self, args, kwargs): + return (self.func_name, repr(args), repr(kwargs)) + + def to_value(self, variables_mapping=None): + """ parse lazy data with evaluated variables mapping. + Notice: variables_mapping should not contain any variable or function. + """ + variables_mapping = variables_mapping or {} + args = parse_lazy_data(self._args, variables_mapping) + kwargs = parse_lazy_data(self._kwargs, variables_mapping) + self.cache_key = self.__prepare_cache_key(args, kwargs) + return self._func(*args, **kwargs) + + +cached_functions_mapping = {} +""" cached function calling results. +""" + + +class LazyString(object): + """ evaluate string lazily. + """ + def __init__(self, raw_string, functions_mapping=None, check_variables_set=None, cached=False): + """ make raw_string as lazy object with functions_mapping + check if any variable undefined in check_variables_set + """ + self.raw_string = raw_string + self.functions_mapping = functions_mapping or {} + self.check_variables_set = check_variables_set or set() + self.cached = cached + self.__parse(raw_string) + + def __parse(self, raw_string): + """ parse raw string, replace function and variable with {} + + Args: + raw_string(str): string with functions or varialbes + e.g. "ABC${func2($a, $b)}DE$c" + + Returns: + string: "ABC{}DE{}" + args: ["${func2($a, $b)}", "$c"] + + """ + self._string = raw_string + args_mapping = {} + + # Notice: functions must be handled before variables + # search function like ${func($a, $b)} + func_match_list = regex_findall_functions(self._string) + match_start_position = 0 + for func_match in func_match_list: + func_str = "${%s(%s)}" % (func_match[0], func_match[1]) + match_start_position = raw_string.index(func_str, match_start_position) + self._string = self._string.replace(func_str, "{}", 1) + function_meta = parse_function_params(func_match[1]) + function_meta = { + "func_name": func_match[0] + } + function_meta.update(parse_function_params(func_match[1])) + lazy_func = LazyFunction( + function_meta, + self.functions_mapping, + self.check_variables_set + ) + args_mapping[match_start_position] = lazy_func + + # search variable like $var + var_match_list = regex_findall_variables(self._string) + match_start_position = 0 + for var_name in var_match_list: + # check if any variable undefined in check_variables_set + if var_name not in self.check_variables_set: + raise exceptions.VariableNotFound(var_name) + + var = "${}".format(var_name) + match_start_position = raw_string.index(var, match_start_position) + # TODO: escape '{' and '}' + # self._string = self._string.replace("{", "{{") + # self._string = self._string.replace("}", "}}") + self._string = self._string.replace(var, "{}", 1) + args_mapping[match_start_position] = var_name + + self._args = [args_mapping[key] for key in sorted(args_mapping.keys())] + + def __repr__(self): + return "LazyString({})".format(self.raw_string) + + def to_value(self, variables_mapping=None): + """ parse lazy data with evaluated variables mapping. + Notice: variables_mapping should not contain any variable or function. + """ + variables_mapping = variables_mapping or {} + + args = [] + for arg in self._args: + if isinstance(arg, LazyFunction): + if self.cached and arg.cache_key and arg.cache_key in cached_functions_mapping: + value = cached_functions_mapping[arg.cache_key] + else: + value = arg.to_value(variables_mapping) + cached_functions_mapping[arg.cache_key] = value + args.append(value) + else: + # variable + var_value = get_mapping_variable(arg, variables_mapping) + args.append(var_value) + + if self._string == "{}": + return args[0] + else: + return self._string.format(*args) + + +def prepare_lazy_data(content, functions_mapping=None, check_variables_set=None, cached=False): + """ make string in content as lazy object with functions_mapping + + Raises: + exceptions.VariableNotFound: if any variable undefined in check_variables_set + """ # TODO: refactor type check if content is None or isinstance(content, (numeric_types, bool, type)): return content - if isinstance(content, (list, set, tuple)): + elif isinstance(content, (list, set, tuple)): return [ - parse_data( + prepare_lazy_data( item, - variables_mapping, functions_mapping, - raise_if_variable_not_found + check_variables_set, + cached ) for item in content ] - if isinstance(content, dict): + elif isinstance(content, dict): parsed_content = {} for key, value in content.items(): - parsed_key = parse_data( + parsed_key = prepare_lazy_data( key, - variables_mapping, functions_mapping, - raise_if_variable_not_found + check_variables_set, + cached ) - parsed_value = parse_data( + parsed_value = prepare_lazy_data( value, - variables_mapping, functions_mapping, - raise_if_variable_not_found + check_variables_set, + cached ) parsed_content[parsed_key] = parsed_value return parsed_content - if isinstance(content, basestring): + elif isinstance(content, basestring): # content is in string format here - variables_mapping = utils.ensure_mapping_format(variables_mapping or {}) - functions_mapping = functions_mapping or {} - content = content.strip() + if not (is_variable_exist(content) or is_function_exist(content)): + # content is neither variable nor function + return content - try: - # replace functions with evaluated value - # Notice: parse_string_functions must be called before parse_string_variables - content = parse_string_functions( - content, - variables_mapping, - functions_mapping - ) - # replace variables with binding value - content = parse_string_variables( - content, - variables_mapping, - functions_mapping - ) - except exceptions.VariableNotFound: - if raise_if_variable_not_found: - raise + functions_mapping = functions_mapping or {} + check_variables_set = check_variables_set or set() + content = content.strip() + content = LazyString(content, functions_mapping, check_variables_set, cached) return content +def parse_lazy_data(content, variables_mapping=None): + """ parse lazy data with evaluated variables mapping. + Notice: variables_mapping should not contain any variable or function. + """ + # TODO: refactor type check + if content is None or isinstance(content, (numeric_types, bool, type)): + return content + + elif isinstance(content, LazyString): + variables_mapping = utils.ensure_mapping_format(variables_mapping or {}) + return content.to_value(variables_mapping) + + elif isinstance(content, (list, set, tuple)): + return [ + parse_lazy_data(item, variables_mapping) + for item in content + ] + + elif isinstance(content, dict): + parsed_content = {} + for key, value in content.items(): + parsed_key = parse_lazy_data(key, variables_mapping) + parsed_value = parse_lazy_data(value, variables_mapping) + parsed_content[parsed_key] = parsed_value + + return parsed_content + + return content + + +def eval_lazy_data(content, variables_mapping=None, functions_mapping=None): + """ evaluate data instantly. + Notice: variables_mapping should not contain any variable or function. + """ + variables_mapping = variables_mapping or {} + check_variables_set = set(variables_mapping.keys()) + return parse_lazy_data( + prepare_lazy_data( + content, + functions_mapping, + check_variables_set + ), + variables_mapping + ) + + +def extract_variables(content): + """ extract all variables in content recursively. + """ + if isinstance(content, (list, set, tuple)): + variables = set() + for item in content: + variables = variables | extract_variables(item) + return variables + + elif isinstance(content, dict): + variables = set() + for key, value in content.items(): + variables = variables | extract_variables(value) + return variables + + elif isinstance(content, LazyString): + return set(regex_findall_variables(content.raw_string)) + + return set() + + +def parse_variables_mapping(variables_mapping, ignore=False): + """ eval each prepared variable and function in variables_mapping. + + Args: + variables_mapping (dict): + { + "varA": LazyString(123$varB), + "varB": LazyString(456$varC), + "varC": LazyString(${sum_two($a, $b)}), + "a": 1, + "b": 2, + "c": {"key": LazyString($b)}, + "d": [LazyString($a), 3] + } + ignore (bool): If set True, VariableNotFound will be ignored. + This is used when initializing tests. + + Returns: + dict: parsed variables_mapping should not contain any variable or function. + { + "varA": "1234563", + "varB": "4563", + "varC": "3", + "a": 1, + "b": 2, + "c": {"key": 2}, + "d": [1, 3] + } + + """ + run_times = 0 + parsed_variables_mapping = {} + + while len(parsed_variables_mapping) != len(variables_mapping): + for var_name in variables_mapping: + + run_times += 1 + if run_times > len(variables_mapping) * 4: + not_found_variables = { + key: variables_mapping[key] + for key in variables_mapping + if key not in parsed_variables_mapping + } + raise exceptions.VariableNotFound(not_found_variables) + + if var_name in parsed_variables_mapping: + continue + + value = variables_mapping[var_name] + variables = extract_variables(value) + + # check if reference variable itself + if var_name in variables: + # e.g. + # var_name = "token" + # variables_mapping = {"token": LazyString($token)} + # var_name = "key" + # variables_mapping = {"key": [LazyString($key), 2]} + if ignore: + parsed_variables_mapping[var_name] = value + continue + raise exceptions.VariableNotFound(var_name) + + if variables: + # reference other variable, or function call with other variable + # e.g. {"varA": "123$varB", "varB": "456$varC"} + # e.g. {"varC": "${sum_two($a, $b)}"} + if any([_var_name not in parsed_variables_mapping for _var_name in variables]): + # reference variable not parsed + continue + + parsed_value = parse_lazy_data(value, parsed_variables_mapping) + parsed_variables_mapping[var_name] = parsed_value + + return parsed_variables_mapping + + def _extend_with_api(test_dict, api_def_dict): """ extend test with api definition, test will merge and override api definition. Args: - test_dict (dict): test block + test_dict (dict): test block, this will override api_def_dict api_def_dict (dict): api definition - Returns: - dict: extended test dict. - Examples: >>> api_def_dict = { "name": "get token 1", @@ -636,6 +724,7 @@ def _extend_with_api(test_dict, api_def_dict): "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] } >>> _extend_with_api(test_dict, api_def_dict) + >>> print(test_dict) { "name": "get token 2", "request": {...}, @@ -644,9 +733,8 @@ def _extend_with_api(test_dict, api_def_dict): } """ - # override name - api_def_name = api_def_dict.pop("name", "") - test_dict["name"] = test_dict.get("name") or api_def_name + # override api name + test_dict.setdefault("name", api_def_dict.pop("name", "api name undefined")) # override variables def_variables = api_def_dict.pop("variables", []) @@ -657,16 +745,12 @@ def _extend_with_api(test_dict, api_def_dict): # merge & override validators TODO: relocate def_raw_validators = api_def_dict.pop("validate", []) - ref_raw_validators = test_dict.get("validate", []) def_validators = [ - parse_validator(validator) - for validator in def_raw_validators + validator.uniform_validator(_validator) + for _validator in def_raw_validators ] - ref_validators = [ - parse_validator(validator) - for validator in ref_raw_validators - ] - test_dict["validate"] = utils.extend_validators( + ref_validators = test_dict.pop("validate", []) + test_dict["validate"] = validator.extend_validators( def_validators, ref_validators ) @@ -678,7 +762,7 @@ def _extend_with_api(test_dict, api_def_dict): test_dict.get("extract", {}) ) - # TODO: merge & override request + # merge & override request test_dict["request"] = api_def_dict.pop("request", {}) # base_url & verify: priority api_def_dict > test_dict @@ -704,8 +788,6 @@ def _extend_with_api(test_dict, api_def_dict): # TODO: extend with other api definition items, e.g. times test_dict.update(api_def_dict) - return test_dict - def _extend_with_testcase(test_dict, testcase_def_dict): """ extend test with testcase definition @@ -732,7 +814,9 @@ def _extend_with_testcase(test_dict, testcase_def_dict): testcase_def_dict["config"]["base_url"] = test_base_url # override name - test_name = test_dict.pop("name") or testcase_def_dict["config"].pop("name") or "Undefined name" + test_name = test_dict.pop("name", None) \ + or testcase_def_dict["config"].pop("name", None) \ + or "testcase name undefined" # override testcase config name, output, etc. testcase_def_dict["config"].update(test_dict) @@ -742,8 +826,8 @@ def _extend_with_testcase(test_dict, testcase_def_dict): test_dict.update(testcase_def_dict) -def __parse_config(config, project_mapping): - """ parse testcase/testsuite config, include variables and name. +def __prepare_config(config, project_mapping, session_variables_set=None): + """ parse testcase/testsuite config. """ # get config variables raw_config_variables = config.pop("variables", {}) @@ -754,39 +838,16 @@ def __parse_config(config, project_mapping): # override config variables with passed in variables raw_config_variables_mapping.update(override_variables) - # parse config variables - parsed_config_variables = {} + if raw_config_variables_mapping: + config["variables"] = raw_config_variables_mapping - for key in raw_config_variables_mapping: - parsed_value = parse_data( - raw_config_variables_mapping[key], - raw_config_variables_mapping, - functions, - raise_if_variable_not_found=False - ) - raw_config_variables_mapping[key] = parsed_value - parsed_config_variables[key] = parsed_value - - if parsed_config_variables: - config["variables"] = parsed_config_variables - - # parse config name - config["name"] = parse_data( - config.get("name", ""), - parsed_config_variables, - functions - ) - - # parse config base_url - if "base_url" in config: - config["base_url"] = parse_data( - config["base_url"], - parsed_config_variables, - functions - ) + check_variables_set = set(raw_config_variables_mapping.keys()) + check_variables_set |= (session_variables_set or set()) + prepared_config = prepare_lazy_data(config, functions, check_variables_set, cached=True) + return prepared_config -def __parse_testcase_tests(tests, config, project_mapping): +def __prepare_testcase_tests(tests, config, project_mapping, session_variables_set=None): """ override tests with testcase config variables, base_url and verify. test maybe nested testcase. @@ -806,50 +867,42 @@ def __parse_testcase_tests(tests, config, project_mapping): """ config_variables = config.get("variables", {}) - config_base_url = config.pop("base_url", "") - config_verify = config.pop("verify", True) + config_base_url = config.get("base_url", "") + config_verify = config.get("verify", True) functions = project_mapping.get("functions", {}) + prepared_testcase_tests = [] + session_variables_set = set(config_variables.keys()) | (session_variables_set or set()) for test_dict in tests: + teststep_variables_set = {"request", "response"} + + # 1, testcase config => testcase tests + # override test_dict variables + test_dict_variables = utils.extend_variables( + test_dict.pop("variables", {}), + config_variables + ) + test_dict["variables"] = test_dict_variables + # base_url & verify: priority test_dict > config if (not test_dict.get("base_url")) and config_base_url: test_dict["base_url"] = config_base_url - # 1, testcase config => testcase tests - # override test_dict variables - test_dict["variables"] = utils.extend_variables( - test_dict.pop("variables", {}), - config_variables - ) - - for key in test_dict["variables"]: - parsed_key = parse_data( - key, - test_dict["variables"], - functions, - raise_if_variable_not_found=False - ) - parsed_value = parse_data( - test_dict["variables"][key], - test_dict["variables"], - functions, - raise_if_variable_not_found=False - ) - if parsed_key in test_dict["variables"]: - test_dict["variables"][parsed_key] = parsed_value - - # parse test_dict name - test_dict["name"] = parse_data( - test_dict.pop("name", ""), - test_dict["variables"], - functions, - raise_if_variable_not_found=False - ) + # unify validators' format + if "validate" in test_dict: + ref_raw_validators = test_dict.pop("validate", []) + test_dict["validate"] = [ + validator.uniform_validator(_validator) + for _validator in ref_raw_validators + ] if "testcase_def" in test_dict: # test_dict is nested testcase + if "output" in test_dict: + session_variables_set |= set(test_dict["output"]) + # 2, testcase test_dict => testcase_def config testcase_def = test_dict.pop("testcase_def") _extend_with_testcase(test_dict, testcase_def) @@ -858,42 +911,62 @@ def __parse_testcase_tests(tests, config, project_mapping): test_dict["config"].setdefault("verify", config_verify) # 3, testcase_def config => testcase_def test_dict - _parse_testcase(test_dict, project_mapping) + test_dict = _parse_testcase(test_dict, project_mapping, session_variables_set) - else: - if "api_def" in test_dict: - # test_dict has API reference - # 2, test_dict => api - api_def_dict = test_dict.pop("api_def") - _extend_with_api(test_dict, api_def_dict) + elif "api_def" in test_dict: + # test_dict has API reference + # 2, test_dict => api + api_def_dict = test_dict.pop("api_def") + _extend_with_api(test_dict, api_def_dict) - if test_dict.get("base_url"): - # parse base_url - base_url = parse_data( - test_dict.pop("base_url"), - test_dict["variables"], - functions - ) - - # build path with base_url - # variable in current url maybe extracted from former api - request_url = parse_data( - test_dict["request"]["url"], - test_dict["variables"], - functions, - raise_if_variable_not_found=False - ) - test_dict["request"]["url"] = utils.build_url( - base_url, - request_url - ) + # current teststep variables + teststep_variables_set |= set(test_dict.get("variables", {}).keys()) # verify priority: testcase teststep > testcase config if "request" in test_dict and "verify" not in test_dict["request"]: test_dict["request"]["verify"] = config_verify + # move extracted variable to session variables + if "extract" in test_dict: + extract_mapping = utils.ensure_mapping_format(test_dict["extract"]) + session_variables_set |= set(extract_mapping.keys()) -def _parse_testcase(testcase, project_mapping): + teststep_variables_set |= session_variables_set + + # convert validators to lazy function + validators = test_dict.pop("validate", []) + prepared_validators = [] + for _validator in validators: + function_meta = { + "func_name": _validator["comparator"], + "args": [ + _validator["check"], + _validator["expect"] + ], + "kwargs": {} + } + prepared_validators.append( + LazyFunction( + function_meta, + functions, + teststep_variables_set + ) + ) + test_dict["validate"] = prepared_validators + + # convert variables and functions to lazy object. + # raises VariableNotFound if undefined variable exists in test_dict + prepared_test_dict = prepare_lazy_data( + test_dict, + functions, + teststep_variables_set + ) + prepared_testcase_tests.append(prepared_test_dict) + + return prepared_testcase_tests + + +def _parse_testcase(testcase, project_mapping, session_variables_set=None): """ parse testcase Args: @@ -905,8 +978,21 @@ def _parse_testcase(testcase, project_mapping): """ testcase.setdefault("config", {}) - __parse_config(testcase["config"], project_mapping) - __parse_testcase_tests(testcase["teststeps"], testcase["config"], project_mapping) + prepared_config = __prepare_config( + testcase["config"], + project_mapping, + session_variables_set + ) + prepared_testcase_tests = __prepare_testcase_tests( + testcase["teststeps"], + prepared_config, + project_mapping, + session_variables_set + ) + return { + "config": prepared_config, + "teststeps": prepared_testcase_tests + } def __get_parsed_testsuite_testcases(testcases, testsuite_config, project_mapping): @@ -922,7 +1008,7 @@ def __get_parsed_testsuite_testcases(testcases, testsuite_config, project_mappin testcases (dict): { "testcase1 name": { - "testcase": "testcases/create_and_check.yml", + "testcase": "testcases/create_user.yml", "weight": 2, "variables": { "uid": 1000 @@ -979,28 +1065,17 @@ def __get_parsed_testsuite_testcases(testcases, testsuite_config, project_mappin # 2, testcase config > testcase_def config # override testcase_def config variables - parsed_testcase_config_variables = utils.extend_variables( + overrided_testcase_config_variables = utils.extend_variables( parsed_testcase["config"].pop("variables", {}), testcase_config_variables ) + if overrided_testcase_config_variables: + parsed_testcase["config"]["variables"] = overrided_testcase_config_variables + # parse config variables - parsed_config_variables = {} - - for key in parsed_testcase_config_variables: - try: - parsed_value = parse_data( - parsed_testcase_config_variables[key], - parsed_testcase_config_variables, - functions - ) - except exceptions.VariableNotFound: - pass - parsed_testcase_config_variables[key] = parsed_value - parsed_config_variables[key] = parsed_value - - if parsed_config_variables: - parsed_testcase["config"]["variables"] = parsed_config_variables + parsed_config_variables = parse_variables_mapping( + overrided_testcase_config_variables, functions) # parse parameters if "parameters" in testcase and testcase["parameters"]: @@ -1012,17 +1087,21 @@ def __get_parsed_testsuite_testcases(testcases, testsuite_config, project_mappin for parameter_variables in cartesian_product_parameters: # deepcopy to avoid influence between parameters - parsed_testcase_copied = utils.deepcopy_dict(parsed_testcase) + testcase_copied = utils.deepcopy_dict(parsed_testcase) parsed_config_variables_copied = utils.deepcopy_dict(parsed_config_variables) - parsed_testcase_copied["config"]["variables"] = utils.extend_variables( + testcase_copied["config"]["variables"] = utils.extend_variables( parsed_config_variables_copied, parameter_variables ) - _parse_testcase(parsed_testcase_copied, project_mapping) + parsed_testcase_copied = _parse_testcase(testcase_copied, project_mapping) + parsed_testcase_copied["config"]["name"] = parse_lazy_data( + parsed_testcase_copied["config"]["name"], + testcase_copied["config"]["variables"] + ) parsed_testcase_list.append(parsed_testcase_copied) else: - _parse_testcase(parsed_testcase, project_mapping) + parsed_testcase = _parse_testcase(parsed_testcase, project_mapping) parsed_testcase_list.append(parsed_testcase) return parsed_testcase_list @@ -1030,10 +1109,10 @@ def __get_parsed_testsuite_testcases(testcases, testsuite_config, project_mappin def _parse_testsuite(testsuite, project_mapping): testsuite.setdefault("config", {}) - __parse_config(testsuite["config"], project_mapping) + prepared_config = __prepare_config(testsuite["config"], project_mapping) parsed_testcase_list = __get_parsed_testsuite_testcases( testsuite["testcases"], - testsuite["config"], + prepared_config, project_mapping ) return parsed_testcase_list @@ -1105,10 +1184,7 @@ def parse_tests(tests_mapping): """ project_mapping = tests_mapping.get("project_mapping", {}) - parsed_tests_mapping = { - "project_mapping": project_mapping, - "testcases": [] - } + testcases = [] for test_type in tests_mapping: @@ -1118,12 +1194,12 @@ def parse_tests(tests_mapping): for testsuite in testsuites: parsed_testcases = _parse_testsuite(testsuite, project_mapping) for parsed_testcase in parsed_testcases: - parsed_tests_mapping["testcases"].append(parsed_testcase) + testcases.append(parsed_testcase) elif test_type == "testcases": for testcase in tests_mapping["testcases"]: - _parse_testcase(testcase, project_mapping) - parsed_tests_mapping["testcases"].append(testcase) + parsed_testcase = _parse_testcase(testcase, project_mapping) + testcases.append(parsed_testcase) elif test_type == "apis": # encapsulate api as a testcase @@ -1131,7 +1207,7 @@ def parse_tests(tests_mapping): testcase = { "teststeps": [api_content] } - _parse_testcase(testcase, project_mapping) - parsed_tests_mapping["testcases"].append(testcase) + parsed_testcase = _parse_testcase(testcase, project_mapping) + testcases.append(parsed_testcase) - return parsed_tests_mapping + return testcases diff --git a/httprunner/runner.py b/httprunner/runner.py index 3cf23987..5df077b5 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -11,27 +11,40 @@ class Runner(object): """ Running testcases. Examples: - >>> functions={...} - >>> config = { - "name": "XXXX", - "base_url": "http://127.0.0.1", - "verify": False + >>> tests_mapping = { + "project_mapping": { + "functions": {} + }, + "testcases": [ + { + "config": { + "name": "XXXX", + "base_url": "http://127.0.0.1", + "verify": False + }, + "teststeps": [ + { + "name": "test description", + "variables": [], # optional + "request": { + "url": "http://127.0.0.1:5000/api/users/1000", + "method": "GET" + } + } + ] + } + ] } - >>> runner = Runner(config, functions) - >>> test_dict = { - "name": "test description", - "variables": [], # optional - "request": { - "url": "http://127.0.0.1:5000/api/users/1000", - "method": "GET" - } - } - >>> runner.run_test(test_dict) + >>> testcases = parser.parse_tests(tests_mapping) + >>> parsed_testcase = testcases[0] + + >>> test_runner = runner.Runner(parsed_testcase["config"]) + >>> test_runner.run_test(parsed_testcase["teststeps"][0]) """ - def __init__(self, config, functions, http_client_session=None): + def __init__(self, config, http_client_session=None): """ run testcase or testsuite. Args: @@ -47,19 +60,18 @@ class Runner(object): http_client_session (instance): requests.Session(), or locust.client.Session() instance. """ - base_url = config.get("base_url") self.verify = config.get("verify", True) self.output = config.get("output", []) - self.functions = functions self.validation_results = [] + config_variables = config.get("variables", {}) # testcase setup hooks testcase_setup_hooks = config.get("setup_hooks", []) # testcase teardown hooks self.testcase_teardown_hooks = config.get("teardown_hooks", []) - self.http_client_session = http_client_session or HttpSession(base_url) - self.session_context = SessionContext(self.functions) + self.http_client_session = http_client_session or HttpSession() + self.session_context = SessionContext(config_variables) if testcase_setup_hooks: self.do_hook_actions(testcase_setup_hooks, "setup") @@ -199,20 +211,24 @@ class Runner(object): self.session_context.init_test_variables(test_variables) # teststep name - test_name = test_dict.get("name", "") + test_name = self.session_context.eval_content(test_dict.get("name", "")) # parse test request raw_request = test_dict.get('request', {}) parsed_test_request = self.session_context.eval_content(raw_request) self.session_context.update_test_variables("request", parsed_test_request) + # prepend url with base_url unless it's already an absolute URL + url = parsed_test_request.pop('url') + base_url = self.session_context.eval_content(test_dict.get("base_url", "")) + parsed_url = utils.build_url(base_url, url) + # setup hooks setup_hooks = test_dict.get("setup_hooks", []) if setup_hooks: self.do_hook_actions(setup_hooks, "setup") try: - url = parsed_test_request.pop('url') method = parsed_test_request.pop('method') parsed_test_request.setdefault("verify", self.verify) group_name = parsed_test_request.pop("group", None) @@ -227,13 +243,13 @@ class Runner(object): logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) - logger.log_info("{method} {url}".format(method=method, url=url)) + logger.log_info("{method} {url}".format(method=method, url=parsed_url)) logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request)) # request resp = self.http_client_session.request( method, - url, + parsed_url, name=(group_name or test_name), **parsed_test_request ) @@ -254,13 +270,12 @@ class Runner(object): validators = test_dict.get("validate", []) try: self.session_context.validate(validators, resp_obj) - except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure): err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32) # log request err_msg += "====== request details ======\n" - err_msg += "url: {}\n".format(url) + err_msg += "url: {}\n".format(parsed_url) err_msg += "method: {}\n".format(method) err_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {})) for k, v in parsed_test_request.items(): @@ -288,7 +303,7 @@ class Runner(object): config = testcase_dict.get("config", {}) # each teststeps in one testcase (YAML/JSON) share the same session. - test_runner = Runner(config, self.functions, self.http_client_session) + test_runner = Runner(config, self.http_client_session) tests = testcase_dict.get("teststeps", []) @@ -351,6 +366,9 @@ class Runner(object): self.meta_datas = None if "teststeps" in test_dict: # nested testcase + test_dict.setdefault("config", {}).setdefault("variables", {}) + test_dict["config"]["variables"].update( + self.session_context.session_variables_mapping) self._run_testcase(test_dict) else: # api diff --git a/httprunner/templates/locustfile_template b/httprunner/templates/locustfile_template index c7582549..410a6fe5 100644 --- a/httprunner/templates/locustfile_template +++ b/httprunner/templates/locustfile_template @@ -15,7 +15,8 @@ logging.getLogger('locust.runners').setLevel(logging.INFO) class WebPageTasks(TaskSet): def on_start(self): - self.test_runner = Runner(self.locust.config, self.locust.functions, self.client) + config = {} + self.test_runner = Runner(config, self.client) @task def test_any(self): @@ -32,14 +33,10 @@ class WebPageTasks(TaskSet): class WebPageUser(HttpLocust): + host = "" task_set = WebPageTasks min_wait = 10 max_wait = 30 file_path = "$TESTCASE_FILE" - locust_tests = prepare_locust_tests(file_path) - functions = locust_tests["functions"] - tests = locust_tests["tests"] - config = {} - - host = config.get('base_url', '') + tests = prepare_locust_tests(file_path) diff --git a/httprunner/utils.py b/httprunner/utils.py index 04f4778b..89e728b5 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -53,7 +53,7 @@ def get_os_environ(variable_name): def build_url(base_url, path): - """ prepend url with hostname unless it's already an absolute URL """ + """ prepend url with base_url unless it's already an absolute URL """ if absolute_http_url_regexp.match(path): return path elif base_url: @@ -120,39 +120,6 @@ def query_json(json_content, query, delimiter='.'): return json_content -def get_uniform_comparator(comparator): - """ convert comparator alias to uniform name - """ - if comparator in ["eq", "equals", "==", "is"]: - return "equals" - elif comparator in ["lt", "less_than"]: - return "less_than" - elif comparator in ["le", "less_than_or_equals"]: - return "less_than_or_equals" - elif comparator in ["gt", "greater_than"]: - return "greater_than" - elif comparator in ["ge", "greater_than_or_equals"]: - return "greater_than_or_equals" - elif comparator in ["ne", "not_equals"]: - return "not_equals" - elif comparator in ["str_eq", "string_equals"]: - return "string_equals" - elif comparator in ["len_eq", "length_equals", "count_eq"]: - return "length_equals" - elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]: - return "length_greater_than" - elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \ - "count_greater_than_or_equals"]: - return "length_greater_than_or_equals" - elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]: - return "length_less_than" - elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \ - "count_less_than_or_equals"]: - return "length_less_than_or_equals" - else: - return comparator - - def deep_update_dict(origin_dict, override_dict): """ update origin dict with override dict recursively e.g. origin_dict = {'a': 1, 'b': {'c': 2, 'd': 4}} @@ -323,78 +290,6 @@ def ensure_mapping_format(variables): raise exceptions.ParamsError("variables format error!") -def _convert_validators_to_mapping(validators): - """ convert validators list to mapping. - - Args: - validators (list): validators in list - - Returns: - dict: validators mapping, use (check, comparator) as key. - - Examples: - >>> validators = [ - {"check": "v1", "expect": 201, "comparator": "eq"}, - {"check": {"b": 1}, "expect": 200, "comparator": "eq"} - ] - >>> _convert_validators_to_mapping(validators) - { - ("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"}, - ('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"} - } - - """ - validators_mapping = {} - - for validator in validators: - if not isinstance(validator["check"], collections.Hashable): - check = json.dumps(validator["check"]) - else: - check = validator["check"] - - key = (check, validator["comparator"]) - validators_mapping[key] = validator - - return validators_mapping - - -def extend_validators(raw_validators, override_validators): - """ extend raw_validators with override_validators. - override_validators will merge and override raw_validators. - - Args: - raw_validators (dict): - override_validators (dict): - - Returns: - list: extended validators - - Examples: - >>> raw_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] - >>> override_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] - >>> extend_validators(raw_validators, override_validators) - [ - {"check": "v1", "expect": 201, "comparator": "eq"}, - {"check": "s2", "expect": 16, "comparator": "len_eq"}, - {"check": "s3", "expect": 12, "comparator": "len_eq"} - ] - - """ - - if not raw_validators: - return override_validators - - elif not override_validators: - return raw_validators - - else: - def_validators_mapping = _convert_validators_to_mapping(raw_validators) - ref_validators_mapping = _convert_validators_to_mapping(override_validators) - - def_validators_mapping.update(ref_validators_mapping) - return list(def_validators_mapping.values()) - - def extend_variables(raw_variables, override_variables): """ extend raw_variables with override_variables. override_variables will merge and override raw_variables. @@ -581,25 +476,6 @@ def gen_cartesian_product(*args): return product_list -def validate_json_file(file_list): - """ validate JSON testcase format - """ - for json_file in set(file_list): - if not json_file.endswith(".json"): - logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file)) - continue - - logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN") - - with io.open(json_file) as stream: - try: - json.load(stream) - except ValueError as e: - raise SystemExit(e) - - print("OK") - - def prettify_json_file(file_list): """ prettify JSON testcase format """ @@ -649,6 +525,13 @@ def omit_long_data(body, omit_len=512): def dump_json_file(json_data, pwd_dir_path, dump_file_name): """ dump json data to file """ + class PythonObjectEncoder(json.JSONEncoder): + def default(self, obj): + try: + return super().default(self, obj) + except TypeError: + return str(obj) + logs_dir_path = os.path.join(pwd_dir_path, "logs") if not os.path.isdir(logs_dir_path): os.makedirs(logs_dir_path) @@ -663,7 +546,8 @@ def dump_json_file(json_data, pwd_dir_path, dump_file_name): json_data, indent=4, separators=(',', ':'), - ensure_ascii=False + ensure_ascii=False, + cls=PythonObjectEncoder )) ) else: @@ -672,14 +556,15 @@ def dump_json_file(json_data, pwd_dir_path, dump_file_name): outfile, indent=4, separators=(',', ':'), - ensure_ascii=False + ensure_ascii=False, + cls=PythonObjectEncoder ) msg = "dump file: {}".format(dump_file_path) logger.color_print(msg, "BLUE") - except TypeError: - msg = "Failed to dump json file: {}".format(dump_file_path) + except TypeError as ex: + msg = "Failed to dump json file: {}\nReason: {}".format(dump_file_path, ex) logger.color_print(msg, "RED") @@ -694,47 +579,18 @@ def _prepare_dump_info(project_mapping, tag_name): return pwd_dir_path, dump_file_name -def dump_tests(tests_mapping, tag_name): - """ dump loaded/parsed tests data (except functions) to json file. +def dump_logs(json_data, project_mapping, tag_name): + """ dump tests data to json file. the dumped file is located in PWD/logs folder. Args: - tests_mapping (dict): data to dump - tag_name (str): tag name, loaded/parsed + json_data (list/dict): json data to dump + project_mapping (dict): project info + tag_name (str): tag name, loaded/parsed/summary """ - project_mapping = tests_mapping.get("project_mapping", {}) pwd_dir_path, dump_file_name = _prepare_dump_info(project_mapping, tag_name) - - tests_to_dump = { - "project_mapping": {} - } - - for key in project_mapping: - if key != "functions": - tests_to_dump["project_mapping"][key] = project_mapping[key] - continue - - # remove functions in order to dump - if project_mapping["functions"]: - debugtalk_py_path = os.path.join(pwd_dir_path, "debugtalk.py") - tests_to_dump["project_mapping"]["debugtalk.py"] = debugtalk_py_path - - if "api" in tests_mapping: - tests_to_dump["api"] = tests_mapping["api"] - elif "testcases" in tests_mapping: - tests_to_dump["testcases"] = tests_mapping["testcases"] - elif "testsuites" in tests_mapping: - tests_to_dump["testsuites"] = tests_mapping["testsuites"] - - dump_json_file(tests_to_dump, pwd_dir_path, dump_file_name) - - -def dump_summary(summary, project_mapping): - """ dump test result summary to json file. - """ - pwd_dir_path, dump_file_name = _prepare_dump_info(project_mapping, "summary") - dump_json_file(summary, pwd_dir_path, dump_file_name) + dump_json_file(json_data, pwd_dir_path, dump_file_name) def get_python2_retire_msg(): diff --git a/httprunner/validator.py b/httprunner/validator.py index 6e52c332..60c15cd1 100644 --- a/httprunner/validator.py +++ b/httprunner/validator.py @@ -1,7 +1,12 @@ # encoding: utf-8 +import collections +import io +import json import os import types +from httprunner import exceptions, logger + """ validate data format TODO: refactor with JSON schema validate @@ -129,6 +134,170 @@ def is_testcase_path(path): return True +############################################################################### +## testcase validator utils +############################################################################### + +def get_uniform_comparator(comparator): + """ convert comparator alias to uniform name + """ + if comparator in ["eq", "equals", "==", "is"]: + return "equals" + elif comparator in ["lt", "less_than"]: + return "less_than" + elif comparator in ["le", "less_than_or_equals"]: + return "less_than_or_equals" + elif comparator in ["gt", "greater_than"]: + return "greater_than" + elif comparator in ["ge", "greater_than_or_equals"]: + return "greater_than_or_equals" + elif comparator in ["ne", "not_equals"]: + return "not_equals" + elif comparator in ["str_eq", "string_equals"]: + return "string_equals" + elif comparator in ["len_eq", "length_equals", "count_eq"]: + return "length_equals" + elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]: + return "length_greater_than" + elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \ + "count_greater_than_or_equals"]: + return "length_greater_than_or_equals" + elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]: + return "length_less_than" + elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \ + "count_less_than_or_equals"]: + return "length_less_than_or_equals" + else: + return comparator + + +def uniform_validator(validator): + """ unify validator + + Args: + validator (dict): validator maybe in two formats: + + format1: this is kept for compatiblity with the previous versions. + {"check": "status_code", "comparator": "eq", "expect": 201} + {"check": "$resp_body_success", "comparator": "eq", "expect": True} + format2: recommended new version, {comparator: [check_item, expected_value]} + {'eq': ['status_code', 201]} + {'eq': ['$resp_body_success', True]} + + Returns + dict: validator info + + { + "check": "status_code", + "expect": 201, + "comparator": "equals" + } + + """ + if not isinstance(validator, dict): + raise exceptions.ParamsError("invalid validator: {}".format(validator)) + + if "check" in validator and "expect" in validator: + # format1 + check_item = validator["check"] + expect_value = validator["expect"] + comparator = validator.get("comparator", "eq") + + elif len(validator) == 1: + # format2 + comparator = list(validator.keys())[0] + compare_values = validator[comparator] + + if not isinstance(compare_values, list) or len(compare_values) != 2: + raise exceptions.ParamsError("invalid validator: {}".format(validator)) + + check_item, expect_value = compare_values + + else: + raise exceptions.ParamsError("invalid validator: {}".format(validator)) + + # uniform comparator, e.g. lt => less_than, eq => equals + comparator = get_uniform_comparator(comparator) + + return { + "check": check_item, + "expect": expect_value, + "comparator": comparator + } + + +def _convert_validators_to_mapping(validators): + """ convert validators list to mapping. + + Args: + validators (list): validators in list + + Returns: + dict: validators mapping, use (check, comparator) as key. + + Examples: + >>> validators = [ + {"check": "v1", "expect": 201, "comparator": "eq"}, + {"check": {"b": 1}, "expect": 200, "comparator": "eq"} + ] + >>> _convert_validators_to_mapping(validators) + { + ("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"}, + ('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"} + } + + """ + validators_mapping = {} + + for validator in validators: + if not isinstance(validator["check"], collections.Hashable): + check = json.dumps(validator["check"]) + else: + check = validator["check"] + + key = (check, validator["comparator"]) + validators_mapping[key] = validator + + return validators_mapping + + +def extend_validators(raw_validators, override_validators): + """ extend raw_validators with override_validators. + override_validators will merge and override raw_validators. + + Args: + raw_validators (dict): + override_validators (dict): + + Returns: + list: extended validators + + Examples: + >>> raw_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] + >>> override_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] + >>> extend_validators(raw_validators, override_validators) + [ + {"check": "v1", "expect": 201, "comparator": "eq"}, + {"check": "s2", "expect": 16, "comparator": "len_eq"}, + {"check": "s3", "expect": 12, "comparator": "len_eq"} + ] + + """ + + if not raw_validators: + return override_validators + + elif not override_validators: + return raw_validators + + else: + def_validators_mapping = _convert_validators_to_mapping(raw_validators) + ref_validators_mapping = _convert_validators_to_mapping(override_validators) + + def_validators_mapping.update(ref_validators_mapping) + return list(def_validators_mapping.values()) + + ############################################################################### ## validate varibles and functions ############################################################################### @@ -157,3 +326,22 @@ def is_variable(tup): return False return True + + +def validate_json_file(file_list): + """ validate JSON testcase format + """ + for json_file in set(file_list): + if not json_file.endswith(".json"): + logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file)) + continue + + logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN") + + with io.open(json_file) as stream: + try: + json.load(stream) + except ValueError as e: + raise SystemExit(e) + + print("OK") diff --git a/tests/api/get_token.yml b/tests/api/get_token.yml index 9444fb9e..24a028bc 100644 --- a/tests/api/get_token.yml +++ b/tests/api/get_token.yml @@ -15,7 +15,7 @@ request: Content-Type: "application/json" device_sn: $device_sn json: - sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)} + sign: ${get_sign($device_sn, $os_platform, $app_version)} validate: - eq: ["status_code", 0] - len_eq: ["content.token", 12] diff --git a/tests/api_server.py b/tests/api_server.py index baadc949..0b20c5d9 100644 --- a/tests/api_server.py +++ b/tests/api_server.py @@ -93,15 +93,13 @@ def index(): @app.route('/api/get-token', methods=['POST']) def get_token(): - user_agent = request.headers.get('User-Agent', "") device_sn = request.headers.get('device_sn', "") os_platform = request.headers.get('os_platform', "") app_version = request.headers.get('app_version', "") data = request.get_json() sign = data.get('sign', "") - expected_sign = get_sign(user_agent, device_sn, os_platform, app_version) - + expected_sign = get_sign(device_sn, os_platform, app_version) if expected_sign != sign: result = { 'success': False, diff --git a/tests/base.py b/tests/base.py index 8c6c5abc..82198842 100644 --- a/tests/base.py +++ b/tests/base.py @@ -50,7 +50,7 @@ class ApiServerUnittest(unittest.TestCase): 'app_version': app_version } data = { - 'sign': get_sign(user_agent, device_sn, os_platform, app_version) + 'sign': get_sign(device_sn, os_platform, app_version) } resp = self.api_client.post(url, json=data, headers=headers) diff --git a/tests/data/bugfix_verify.yml b/tests/data/bugfix_verify.yml new file mode 100644 index 00000000..d3cd0c47 --- /dev/null +++ b/tests/data/bugfix_verify.yml @@ -0,0 +1,13 @@ +- config: + name: basic test with httpbin + base_url: https://httpbin.org/ + verify: False + +- test: + name: headers + request: + url: /headers + method: GET + validate: + - eq: ["status_code", 200] + - eq: [content.headers.Host, "httpbin.org"] diff --git a/tests/data/demo_testcase.yml b/tests/data/demo_testcase.yml index 36924cf3..2a495b98 100644 --- a/tests/data/demo_testcase.yml +++ b/tests/data/demo_testcase.yml @@ -1,8 +1,9 @@ - config: - name: "123$var_a" + name: "123t$var_a" variables: - var_a: 0 - var_c: "${sum_two(1, 2)}" + var_a: 1 + var_b: 2 + var_c: "${sum_two($var_a, $var_b)}" var_d: "${gen_random_string(5)}" var_e: $var_d PROJECT_KEY: ${ENV(PROJECT_KEY)} diff --git a/tests/data/demo_testcase_cli.yml b/tests/data/demo_testcase_cli.yml index 56cb0ef3..27db9390 100644 --- a/tests/data/demo_testcase_cli.yml +++ b/tests/data/demo_testcase_cli.yml @@ -10,7 +10,7 @@ os_platform: 'ios' app_version: '2.8.6' json: - sign: f1219719911caae89ccc301679857ebfda115ca2 + sign: 5188962c489d1a35effa99e9346dd5efd4fdabad variables: expect_status_code: 200 token_len: 16 diff --git a/tests/data/demo_testcase_functions.yml b/tests/data/demo_testcase_functions.yml index 43b1c55f..801b618b 100644 --- a/tests/data/demo_testcase_functions.yml +++ b/tests/data/demo_testcase_functions.yml @@ -18,7 +18,7 @@ os_platform: $os_platform app_version: $app_version json: - sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)} + sign: ${get_sign($device_sn, $os_platform, $app_version)} extract: - token: content.token validate: diff --git a/tests/data/demo_testcase_hardcode.json b/tests/data/demo_testcase_hardcode.json index 690058f9..efe42738 100644 --- a/tests/data/demo_testcase_hardcode.json +++ b/tests/data/demo_testcase_hardcode.json @@ -13,7 +13,7 @@ "app_version": "2.8.6" }, "json": { - "sign": "f1219719911caae89ccc301679857ebfda115ca2" + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" } }, "variables": [ diff --git a/tests/data/demo_testcase_hardcode.yml b/tests/data/demo_testcase_hardcode.yml index 004d2ac2..05df6b4a 100644 --- a/tests/data/demo_testcase_hardcode.yml +++ b/tests/data/demo_testcase_hardcode.yml @@ -10,7 +10,7 @@ os_platform: 'ios' app_version: '2.8.6' json: - sign: f1219719911caae89ccc301679857ebfda115ca2 + sign: 5188962c489d1a35effa99e9346dd5efd4fdabad variables: expect_status_code: 200 token_len: 16 diff --git a/tests/data/demo_testcase_variables.yml b/tests/data/demo_testcase_variables.yml index 3119b1fa..2510d514 100644 --- a/tests/data/demo_testcase_variables.yml +++ b/tests/data/demo_testcase_variables.yml @@ -10,7 +10,7 @@ user_agent: 'iOS/10.3' os_platform: 'ios' app_version: '2.8.6' - sign: f1219719911caae89ccc301679857ebfda115ca2 + sign: 5188962c489d1a35effa99e9346dd5efd4fdabad request: url: /api/get-token method: POST diff --git a/tests/httpbin/api/302_redirect.yml b/tests/httpbin/api/302_redirect.yml index 8b24637f..959cc227 100644 --- a/tests/httpbin/api/302_redirect.yml +++ b/tests/httpbin/api/302_redirect.yml @@ -6,5 +6,6 @@ request: url: https://debugtalk.com status_code: 302 method: GET + verify: False validate: - eq: ["status_code", 200] diff --git a/tests/locust_tests/demo_locusts.yml b/tests/locust_tests/demo_locusts.yml index 352d4932..4ae14cde 100644 --- a/tests/locust_tests/demo_locusts.yml +++ b/tests/locust_tests/demo_locusts.yml @@ -6,13 +6,13 @@ config: testcases: create user 1000 and check result.: - testcase: testcases/create_and_check.yml + testcase: testcases/create_user.yml weight: 2 variables: uid: 1000 create user 1001 and check result.: - testcase: testcases/create_and_check.yml + testcase: testcases/create_user.yml weight: 3 variables: uid: 1001 diff --git a/tests/test_api.py b/tests/test_api.py index b0b240f3..db13b543 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -36,7 +36,7 @@ class TestHttpRunner(ApiServerUnittest): 'url': 'http://127.0.0.1:5000/api/get-token', 'method': 'POST', 'headers': {'Content-Type': 'application/json', 'app_version': '2.8.6', 'device_sn': 'FwgRiO7CNA50DSU', 'os_platform': 'ios', 'user_agent': 'iOS/10.3'}, - 'json': {'sign': '958a05393efef0ac7c0fb80a7eac45e24fd40c27'} + 'json': {'sign': '9c0c7e51c91ae963c833a4ccbab8d683c4a90c98'} }, 'extract': [ {'token': 'content.token'} @@ -52,7 +52,8 @@ class TestHttpRunner(ApiServerUnittest): 'request': { 'url': 'http://127.0.0.1:5000/api/users/1000', 'method': 'POST', - 'headers': {'Content-Type': 'application/json', 'device_sn': 'FwgRiO7CNA50DSU','token': '$token'}, 'json': {'name': 'user1', 'password': '123456'} + 'headers': {'Content-Type': 'application/json', 'device_sn': 'FwgRiO7CNA50DSU','token': '$token'}, + 'json': {'name': 'user1', 'password': '123456'} }, 'validate': [ {'eq': ['status_code', 201]}, @@ -137,11 +138,11 @@ class TestHttpRunner(ApiServerUnittest): self.assertEqual(len(vars_out), 6) self.assertEqual(vars_out[0]["in"]["uid"], 101) self.assertEqual(vars_out[0]["in"]["device_sn"], "TESTSUITE_X1") - token1 = vars_out[0]["out"]["token"] + token1 = vars_out[0]["out"]["session_token"] self.assertEqual(len(token1), 16) self.assertEqual(vars_out[5]["in"]["uid"], 103) self.assertEqual(vars_out[5]["in"]["device_sn"], "TESTSUITE_X2") - token2 = vars_out[0]["out"]["token"] + token2 = vars_out[0]["out"]["session_token"] self.assertEqual(len(token2), 16) self.assertEqual(token1, token2) @@ -240,7 +241,7 @@ class TestHttpRunner(ApiServerUnittest): summary = self.runner.summary self.assertTrue(summary["success"]) self.assertEqual(summary["stat"]["testcases"]["total"], 2) - self.assertEqual(summary["stat"]["teststeps"]["total"], 8) + self.assertEqual(summary["stat"]["teststeps"]["total"], 4) def test_run_httprunner_with_hooks(self): testcase_file_path = os.path.join( @@ -470,7 +471,7 @@ class TestHttpRunner(ApiServerUnittest): self.assertEqual(len(summary["details"]), 3 * 2) self.assertEqual(summary["stat"]["testcases"]["total"], 6) - self.assertEqual(summary["stat"]["teststeps"]["total"], 3 * 2 * 4) + self.assertEqual(summary["stat"]["teststeps"]["total"], 3 * 2 * 2) self.assertEqual( summary["details"][0]["name"], "create user 101 and check result for TESTSUITE_X1." @@ -481,10 +482,10 @@ class TestHttpRunner(ApiServerUnittest): ) self.assertEqual( summary["details"][0]["stat"]["total"], - 4 + 2 ) records_name_list = [ - summary["details"][i]["records"][2]["name"] + summary["details"][i]["records"][1]["meta_datas"][1]["name"] for i in range(6) ] self.assertEqual( @@ -577,8 +578,7 @@ class TestApi(ApiServerUnittest): testcase_path = "tests/testcases/setup.yml" tests_mapping = loader.load_tests(testcase_path) - parsed_tests_mapping = parser.parse_tests(tests_mapping) - parsed_testcases = parsed_tests_mapping["testcases"] + parsed_testcases = parser.parse_tests(tests_mapping) self.assertEqual(len(parsed_testcases), 1) @@ -589,7 +589,7 @@ class TestApi(ApiServerUnittest): self.assertEqual(test_dict1["name"], "get token (setup)") self.assertNotIn("api_def", test_dict1) self.assertEqual(test_dict1["variables"]["device_sn"], "TESTCASE_SETUP_XXX") - self.assertEqual(test_dict1["request"]["url"], "http://127.0.0.1:5000/api/get-token") + self.assertEqual(test_dict1["request"]["url"], "/api/get-token") self.assertEqual(test_dict1["request"]["verify"], False) test_dict2 = parsed_testcases[0]["teststeps"][1] @@ -599,9 +599,9 @@ class TestApi(ApiServerUnittest): testcase_path = "tests/testcases/setup.yml" tests_mapping = loader.load_tests(testcase_path) - parsed_tests_mapping = parser.parse_tests(tests_mapping) + testcases = parser.parse_tests(tests_mapping) runner = HttpRunner() - test_suite = runner._add_tests(parsed_tests_mapping) + test_suite = runner._add_tests(testcases) self.assertEqual(len(test_suite._tests), 1) teststeps = test_suite._tests[0].teststeps @@ -610,38 +610,39 @@ class TestApi(ApiServerUnittest): self.assertIn("api", teststeps[0]) def test_testcase_complex_verify(self): - testcase_path = "tests/testcases/create_and_check.yml" + testcase_path = "tests/testcases/create_user.yml" tests_mapping = loader.load_tests(testcase_path) - parsed_tests_mapping = parser.parse_tests(tests_mapping) - teststeps = parsed_tests_mapping["testcases"][0]["teststeps"] + testcases = parser.parse_tests(tests_mapping) + teststeps = testcases[0]["teststeps"] # testcases/setup.yml - teststep1 = teststeps[0] - self.assertEqual(teststep1["teststeps"][0]["request"]["verify"], False) - self.assertEqual(teststep1["teststeps"][1]["request"]["verify"], False) + teststep0 = teststeps[0] + self.assertEqual(teststep0["teststeps"][0]["request"]["verify"], False) + self.assertEqual(teststep0["teststeps"][1]["request"]["verify"], False) - # testcases/create_and_check.yml teststep 2/3/4 - self.assertEqual(teststeps[1]["request"]["verify"], True) - self.assertEqual(teststeps[2]["request"]["verify"], True) - self.assertEqual(teststeps[3]["request"]["verify"], True) + # testcases/create_user.yml + teststep1 = teststeps[1] + self.assertEqual(teststep1["teststeps"][0]["request"]["verify"], True) + self.assertEqual(teststep1["teststeps"][1]["request"]["verify"], True) + self.assertEqual(teststep1["teststeps"][2]["request"]["verify"], True) def test_testcase_simple_run_suite(self): testcase_path = "tests/testcases/setup.yml" tests_mapping = loader.load_tests(testcase_path) - parsed_tests_mapping = parser.parse_tests(tests_mapping) + testcases = parser.parse_tests(tests_mapping) runner = HttpRunner() - test_suite = runner._add_tests(parsed_tests_mapping) + test_suite = runner._add_tests(testcases) tests_results = runner._run_suite(test_suite) self.assertEqual(len(tests_results[0][1].records), 2) def test_testcase_complex_run_suite(self): - testcase_path = "tests/testcases/create_and_check.yml" + testcase_path = "tests/testcases/create_user.yml" tests_mapping = loader.load_tests(testcase_path) - parsed_tests_mapping = parser.parse_tests(tests_mapping) + testcases = parser.parse_tests(tests_mapping) runner = HttpRunner() - test_suite = runner._add_tests(parsed_tests_mapping) + test_suite = runner._add_tests(testcases) tests_results = runner._run_suite(test_suite) - self.assertEqual(len(tests_results[0][1].records), 4) + self.assertEqual(len(tests_results[0][1].records), 2) results = tests_results[0][1] self.assertEqual( @@ -650,7 +651,7 @@ class TestApi(ApiServerUnittest): ) self.assertEqual( results.records[1]["name"], - "make sure user 9001 does not exist" + "create user and check result." ) def test_testsuite_loader(self): @@ -679,7 +680,7 @@ class TestApi(ApiServerUnittest): self.assertEqual(testcase_tests["name"], "create user 1000 and check result.") self.assertIsInstance(testcase_tests["testcase_def"], dict) self.assertEqual(testcase_tests["testcase_def"]["config"]["name"], "create user and check result.") - self.assertEqual(len(testcase_tests["testcase_def"]["teststeps"]), 4) + self.assertEqual(len(testcase_tests["testcase_def"]["teststeps"]), 2) self.assertEqual( testcase_tests["testcase_def"]["teststeps"][0]["name"], "setup and reset all (override) for $device_sn." @@ -689,57 +690,52 @@ class TestApi(ApiServerUnittest): testcase_path = "tests/testsuites/create_users.yml" tests_mapping = loader.load_tests(testcase_path) - parsed_tests_mapping = parser.parse_tests(tests_mapping) - - parsed_testcases = parsed_tests_mapping["testcases"] + parsed_testcases = parser.parse_tests(tests_mapping) self.assertEqual(len(parsed_testcases), 2) - self.assertEqual(len(parsed_testcases[0]["teststeps"]), 4) + self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2) testcase1 = parsed_testcases[0]["teststeps"][0] - self.assertIn("setup and reset all (override)", testcase1["config"]["name"]) - self.assertEqual(testcase1["teststeps"][0]["variables"]["var_c"], testcase1["teststeps"][0]["variables"]["var_d"]) - self.assertEqual(testcase1["teststeps"][0]["variables"]["var_a"], testcase1["teststeps"][0]["variables"]["var_b"]) - self.assertNotEqual(testcase1["teststeps"][0]["variables"]["var_a"], testcase1["teststeps"][0]["variables"]["var_c"]) + self.assertIn("setup and reset all (override)", testcase1["config"]["name"].raw_string) + teststeps = testcase1["teststeps"] self.assertNotIn("testcase_def", testcase1) - self.assertEqual(len(testcase1["teststeps"]), 2) + self.assertEqual(len(teststeps), 2) self.assertEqual( - testcase1["teststeps"][0]["request"]["url"], - "http://127.0.0.1:5000/api/get-token" + teststeps[0]["request"]["url"], + "/api/get-token" ) - self.assertEqual(len(testcase1["teststeps"][0]["variables"]["device_sn"]), 15) def test_testsuite_add_tests(self): testcase_path = "tests/testsuites/create_users.yml" tests_mapping = loader.load_tests(testcase_path) - parsed_tests_mapping = parser.parse_tests(tests_mapping) + testcases = parser.parse_tests(tests_mapping) runner = HttpRunner() - test_suite = runner._add_tests(parsed_tests_mapping) + test_suite = runner._add_tests(testcases) self.assertEqual(len(test_suite._tests), 2) tests = test_suite._tests[0].teststeps - self.assertIn("setup and reset all (override)", tests[0]["config"]["name"]) + self.assertIn("setup and reset all (override)", tests[0]["config"]["name"].raw_string) def test_testsuite_run_suite(self): testcase_path = "tests/testsuites/create_users.yml" tests_mapping = loader.load_tests(testcase_path) - parsed_tests_mapping = parser.parse_tests(tests_mapping) + testcases = parser.parse_tests(tests_mapping) runner = HttpRunner() - test_suite = runner._add_tests(parsed_tests_mapping) + test_suite = runner._add_tests(testcases) tests_results = runner._run_suite(test_suite) - self.assertEqual(len(tests_results[0][1].records), 4) + self.assertEqual(len(tests_results[0][1].records), 2) results = tests_results[0][1] self.assertIn( "setup and reset all (override)", results.records[0]["name"] ) - self.assertIn( + self.assertEqual( results.records[1]["name"], - ["make sure user 1000 does not exist", "make sure user 1001 does not exist"] + "create user and check result." ) @@ -749,11 +745,10 @@ class TestLocust(unittest.TestCase): path = os.path.join( os.getcwd(), 'tests/locust_tests/demo_locusts.yml') locust_tests = prepare_locust_tests(path) - self.assertIn("gen_md5", locust_tests["functions"]) - self.assertEqual(len(locust_tests["tests"]), 2 + 3) + self.assertEqual(len(locust_tests), 2 + 3) name_list = [ "create user 1000 and check result.", "create user 1001 and check result." ] - self.assertIn(locust_tests["tests"][0]["config"]["name"], name_list) - self.assertIn(locust_tests["tests"][4]["config"]["name"], name_list) + self.assertIn(locust_tests[0]["config"]["name"], name_list) + self.assertIn(locust_tests[4]["config"]["name"], name_list) diff --git a/tests/test_client.py b/tests/test_client.py index 9be921b5..70d4d276 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -7,7 +7,7 @@ from tests.base import ApiServerUnittest class TestHttpClient(ApiServerUnittest): def setUp(self): super(TestHttpClient, self).setUp() - self.api_client = HttpSession(self.host) + self.api_client = HttpSession() self.headers = self.get_authenticated_headers() self.reset_all() @@ -30,7 +30,7 @@ class TestHttpClient(ApiServerUnittest): self.assertEqual(True, resp.json()['success']) def test_request_without_base_url(self): - url = "/api/users/1000" + url = "{}/api/users/1000".format(self.host) data = { 'name': 'user1', 'password': '123456' @@ -40,7 +40,7 @@ class TestHttpClient(ApiServerUnittest): self.assertEqual(True, resp.json()['success']) def test_request_post_data(self): - url = "/api/users/1000" + url = "{}/api/users/1000".format(self.host) data = { 'name': 'user1', 'password': '123456' @@ -56,7 +56,7 @@ class TestHttpClient(ApiServerUnittest): self.assertIn("password=123456", resp.request.body) def test_request_with_cookies(self): - url = "/api/users/1000" + url = "{}/api/users/1000".format(self.host) data = { 'name': 'user1', 'password': '123456' @@ -76,7 +76,7 @@ class TestHttpClient(ApiServerUnittest): "a": "1", "b": "2" } - resp = self.api_client.get(url, cookies=cookies, headers=self.headers) + resp = self.api_client.get(url, cookies=cookies, headers=self.headers, verify=False) raw_request = resp.history[0].request self.assertEqual(raw_request._cookies["a"], "1") self.assertEqual(raw_request._cookies["b"], "2") diff --git a/tests/test_context.py b/tests/test_context.py index a5c6a835..6659ef24 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,9 +1,8 @@ import os import time -import requests -from httprunner import context, exceptions, loader, response, utils -from tests.base import ApiServerUnittest +from httprunner import context, exceptions, loader, parser, runner +from tests.base import ApiServerUnittest, gen_md5, gen_random_string class TestContext(ApiServerUnittest): @@ -12,14 +11,9 @@ class TestContext(ApiServerUnittest): loader.load_project_tests(os.path.join(os.getcwd(), "tests")) project_mapping = loader.project_mapping self.context = context.SessionContext( - functions=project_mapping["functions"], variables={"SECRET_KEY": "DebugTalk"} ) - def test_init_context_functions(self): - context_functions = self.context.FUNCTIONS_MAPPING - self.assertIn("gen_md5", context_functions) - def test_init_test_variables_initialize(self): self.assertEqual( self.context.test_variables_mapping, @@ -30,16 +24,24 @@ class TestContext(ApiServerUnittest): variables = { "random": "${gen_random_string($num)}", "authorization": "${gen_md5($TOKEN, $data, $random)}", - "data": '{"name": "$username", "password": "123456"}', + "data": "$username", + # TODO: escape '{' and '}' + # "data": '{"name": "$username", "password": "123456"}', "TOKEN": "debugtalk", "username": "user1", "num": 6 } + functions = { + "gen_random_string": gen_random_string, + "gen_md5": gen_md5 + } + variables = parser.prepare_lazy_data(variables, functions, variables.keys()) + variables = parser.parse_variables_mapping(variables) self.context.init_test_variables(variables) variables_mapping = self.context.test_variables_mapping self.assertEqual(len(variables_mapping["random"]), 6) self.assertEqual(len(variables_mapping["authorization"]), 32) - self.assertEqual(variables_mapping["data"], '{"name": "user1", "password": "123456"}') + self.assertEqual(variables_mapping["data"], 'user1') def test_update_seesion_variables(self): self.context.update_session_variables({"TOKEN": "debugtalk"}) @@ -48,15 +50,11 @@ class TestContext(ApiServerUnittest): "debugtalk" ) - def test_eval_content_functions(self): - content = "${sleep_N_secs(1)}" - start_time = time.time() - self.context.eval_content(content) - elapsed_time = time.time() - start_time - self.assertGreater(elapsed_time, 1) - def test_eval_content_variables(self): - content = "abc$SECRET_KEY" + variables = { + "SECRET_KEY": "DebugTalk" + } + content = parser.prepare_lazy_data("abc$SECRET_KEY", {}, variables.keys()) self.assertEqual( self.context.eval_content(content), "abcDebugTalk" @@ -76,7 +74,12 @@ class TestContext(ApiServerUnittest): "authorization": "${gen_md5($TOKEN, $data, $random)}", "TOKEN": "debugtalk" } - + functions = { + "gen_random_string": gen_random_string, + "gen_md5": gen_md5 + } + variables = parser.prepare_lazy_data(variables, functions, variables.keys()) + variables = parser.parse_variables_mapping(variables) self.context.init_test_variables(variables) request = { @@ -90,7 +93,12 @@ class TestContext(ApiServerUnittest): }, "data": "$data" } - parsed_request = self.context.eval_content(request) + prepared_request = parser.prepare_lazy_data( + request, + functions, + {"authorization", "random", "SECRET_KEY", "data"} + ) + parsed_request = self.context.eval_content(prepared_request) self.assertIn("authorization", parsed_request["headers"]) self.assertEqual(len(parsed_request["headers"]["authorization"]), 32) self.assertIn("random", parsed_request["headers"]) @@ -102,74 +110,80 @@ class TestContext(ApiServerUnittest): ) self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk") - def test_do_validation(self): - self.context._do_validation( - {"check": "check", "check_value": 1, "expect": 1, "comparator": "eq"} - ) - self.context._do_validation( - {"check": "check", "check_value": "abc", "expect": "abc", "comparator": "=="} - ) - self.context._do_validation( - {"check": "status_code", "check_value": "201", "expect": 3, "comparator": "sum_status_code"} - ) - def test_validate(self): - url = "http://127.0.0.1:5000/" - resp = requests.get(url) - resp_obj = response.ResponseObject(resp) - - validators = [ - {"eq": ["$resp_status_code", 201]}, - {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, - {"check": "$resp_body_success", "comparator": "eq", "expect": True} + testcases = [ + { + "config": { + 'name': "test validation" + }, + "teststeps": [ + { + "name": "test validation", + "request": { + "url": "http://127.0.0.1:5000/", + "method": "GET", + }, + "variables": { + "resp_status_code": 200, + "resp_body_success": True + }, + "validate": [ + {"eq": ["$resp_status_code", 200]}, + {"check": "$resp_status_code", "comparator": "eq", "expect": 200}, + {"check": "$resp_body_success", "expect": True}, + {"check": "${is_status_code_200($resp_status_code)}", "expect": True} + ] + } + ] + } ] - variables = { - "resp_status_code": 200, - "resp_body_success": True + from tests.debugtalk import is_status_code_200 + tests_mapping = { + "project_mapping": { + "functions": { + "is_status_code_200": is_status_code_200 + } + }, + "testcases": testcases } - - self.context.init_test_variables(variables) - - with self.assertRaises(exceptions.ValidationFailure): - self.context.validate(validators, resp_obj) - - validators = [ - {"eq": ["$resp_status_code", 201]}, - {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, - {"check": "$resp_body_success", "comparator": "eq", "expect": True}, - {"check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False} - ] - variables = [ - {"resp_status_code": 201}, - {"resp_body_success": True} - ] - self.context.init_test_variables(variables) - self.context.validate(validators, resp_obj) - - self.context.validate([], resp_obj) - self.assertEqual(self.context.validation_results, []) + testcases = parser.parse_tests(tests_mapping) + parsed_testcase = testcases[0] + test_runner = runner.Runner(parsed_testcase["config"]) + teststep = parsed_testcase["teststeps"][0] + test_runner.run_test(teststep) def test_validate_exception(self): - url = "http://127.0.0.1:5000/" - resp = requests.get(url) - resp_obj = response.ResponseObject(resp) - - # expected value missed in validators - validators = [ - {"eq": ["$resp_status_code", 201]}, - {"check": "$resp_status_code", "comparator": "eq", "expect": 201} + testcases = [ + { + "config": { + 'name': "test validation" + }, + "teststeps": [ + { + "name": "test validation", + "request": { + "url": "http://127.0.0.1:5000/", + "method": "GET", + }, + "variables": { + "resp_status_code": 200, + "resp_body_success": True + }, + "validate": [ + {"eq": ["$resp_status_code", 201]}, + {"check": "$resp_status_code", "expect": 201}, + {"check": "$resp_body_success", "comparator": "eq", "expect": True} + ] + } + ] + } ] - variables = [] - self.context.init_test_variables(variables) - - with self.assertRaises(exceptions.VariableNotFound): - self.context.validate(validators, resp_obj) - - # expected value missed in variables mapping - variables = [ - {"resp_status_code": 200} - ] - self.context.init_test_variables(variables) - + tests_mapping = { + "testcases": testcases + } + testcases = parser.parse_tests(tests_mapping) + parsed_testcase = testcases[0] + test_runner = runner.Runner(parsed_testcase["config"]) + teststep = parsed_testcase["teststeps"][0] with self.assertRaises(exceptions.ValidationFailure): - self.context.validate(validators, resp_obj) + test_runner.run_test(teststep) diff --git a/tests/test_loader.py b/tests/test_loader.py index 10d79cdf..97845ddd 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -348,14 +348,14 @@ class TestSuiteLoader(unittest.TestCase): tests_mapping = loader.load_tests(testcase_file_path) testcases = tests_mapping["testcases"] self.assertIsInstance(testcases, list) - self.assertEqual(testcases[0]["config"]["name"], '123$var_a') + self.assertEqual(testcases[0]["config"]["name"], '123t$var_a') self.assertIn( "sum_two", tests_mapping["project_mapping"]["functions"] ) self.assertEqual( testcases[0]["config"]["variables"]["var_c"], - "${sum_two(1, 2)}" + "${sum_two($var_a, $var_b)}" ) self.assertEqual( testcases[0]["config"]["variables"]["PROJECT_KEY"], diff --git a/tests/test_parser.py b/tests/test_parser.py index 663de616..fe5ad205 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,11 +1,13 @@ import os +import re import time import unittest from httprunner import exceptions, loader, parser +from tests.debugtalk import gen_random_string, sum_two -class TestParser(unittest.TestCase): +class TestParserBasic(unittest.TestCase): def test_parse_string_value(self): self.assertEqual(parser.parse_string_value("123"), 123) @@ -14,139 +16,170 @@ class TestParser(unittest.TestCase): self.assertEqual(parser.parse_string_value("$var"), "$var") self.assertEqual(parser.parse_string_value("${func}"), "${func}") - def test_extract_variables(self): + def test_regex_findall_variables(self): self.assertEqual( - parser.extract_variables("$var"), + parser.regex_findall_variables("$var"), ["var"] ) self.assertEqual( - parser.extract_variables("$var123"), + parser.regex_findall_variables("$var123"), ["var123"] ) self.assertEqual( - parser.extract_variables("$var_name"), + parser.regex_findall_variables("$var_name"), ["var_name"] ) self.assertEqual( - parser.extract_variables("var"), + parser.regex_findall_variables("var"), [] ) self.assertEqual( - parser.extract_variables("a$var"), + parser.regex_findall_variables("a$var"), ["var"] ) self.assertEqual( - parser.extract_variables("$v ar"), + parser.regex_findall_variables("$v ar"), ["v"] ) self.assertEqual( - parser.extract_variables(" "), + parser.regex_findall_variables(" "), [] ) self.assertEqual( - parser.extract_variables("$abc*"), + parser.regex_findall_variables("$abc*"), ["abc"] ) self.assertEqual( - parser.extract_variables("${func()}"), + parser.regex_findall_variables("${func()}"), [] ) self.assertEqual( - parser.extract_variables("${func(1,2)}"), + parser.regex_findall_variables("${func(1,2)}"), [] ) self.assertEqual( - parser.extract_variables("${gen_md5($TOKEN, $data, $random)}"), + parser.regex_findall_variables("${gen_md5($TOKEN, $data, $random)}"), ["TOKEN", "data", "random"] ) - def test_parse_function(self): + def test_parse_function_params(self): self.assertEqual( - parser.parse_function("func()"), - {'func_name': 'func', 'args': [], 'kwargs': {}} + parser.parse_function_params(""), + {'args': [], 'kwargs': {}} ) self.assertEqual( - parser.parse_function("func(5)"), - {'func_name': 'func', 'args': [5], 'kwargs': {}} + parser.parse_function_params("5"), + {'args': [5], 'kwargs': {}} ) self.assertEqual( - parser.parse_function("func(1, 2)"), - {'func_name': 'func', 'args': [1, 2], 'kwargs': {}} + parser.parse_function_params("1, 2"), + {'args': [1, 2], 'kwargs': {}} ) self.assertEqual( - parser.parse_function("func(a=1, b=2)"), - {'func_name': 'func', 'args': [], 'kwargs': {'a': 1, 'b': 2}} + parser.parse_function_params("a=1, b=2"), + {'args': [], 'kwargs': {'a': 1, 'b': 2}} ) self.assertEqual( - parser.parse_function("func(a= 1, b =2)"), - {'func_name': 'func', 'args': [], 'kwargs': {'a': 1, 'b': 2}} + parser.parse_function_params("a= 1, b =2"), + {'args': [], 'kwargs': {'a': 1, 'b': 2}} ) self.assertEqual( - parser.parse_function("func(1, 2, a=3, b=4)"), - {'func_name': 'func', 'args': [1, 2], 'kwargs': {'a': 3, 'b': 4}} + parser.parse_function_params("1, 2, a=3, b=4"), + {'args': [1, 2], 'kwargs': {'a': 3, 'b': 4}} ) self.assertEqual( - parser.parse_function("func($request, 123)"), - {'func_name': 'func', 'args': ["$request", 123], 'kwargs': {}} + parser.parse_function_params("$request, 123"), + {'args': ["$request", 123], 'kwargs': {}} ) self.assertEqual( - parser.parse_function("func( )"), - {'func_name': 'func', 'args': [], 'kwargs': {}} + parser.parse_function_params(" "), + {'args': [], 'kwargs': {}} ) self.assertEqual( - parser.parse_function("func(hello world, a=3, b=4)"), - {'func_name': 'func', 'args': ["hello world"], 'kwargs': {'a': 3, 'b': 4}} + parser.parse_function_params("hello world, a=3, b=4"), + {'args': ["hello world"], 'kwargs': {'a': 3, 'b': 4}} ) self.assertEqual( - parser.parse_function("func($request, 12 3)"), - {'func_name': 'func', 'args': ["$request", '12 3'], 'kwargs': {}} + parser.parse_function_params("$request, 12 3"), + {'args': ["$request", '12 3'], 'kwargs': {}} ) - def test_parse_validator(self): - validator = {"check": "status_code", "comparator": "eq", "expect": 201} + def test_extract_variables(self): + prepared_content = parser.prepare_lazy_data("123$a", {}, {"a"}) self.assertEqual( - parser.parse_validator(validator), - {"check": "status_code", "comparator": "eq", "expect": 201} + parser.extract_variables(prepared_content), + {"a"} ) - - validator = {'eq': ['status_code', 201]} + prepared_content = parser.prepare_lazy_data("$a$b", {}, {"a", "b"}) self.assertEqual( - parser.parse_validator(validator), - {"check": "status_code", "comparator": "eq", "expect": 201} + parser.extract_variables(prepared_content), + {"a", "b"} + ) + prepared_content = parser.prepare_lazy_data(["$a$b", "$c", "d"], {}, {"a", "b", "c", "d"}) + self.assertEqual( + parser.extract_variables(prepared_content), + {"a", "b", "c"} + ) + prepared_content = parser.prepare_lazy_data( + {"a": 1, "b": {"c": "$d", "e": 3}}, + {}, + {"d"} + ) + self.assertEqual( + parser.extract_variables(prepared_content), + {"d"} + ) + prepared_content = parser.prepare_lazy_data( + {"a": ["$b"], "b": {"c": "$d", "e": 3}}, + {}, + {"b", "d"} + ) + self.assertEqual( + parser.extract_variables(prepared_content), + {"b", "d"} + ) + prepared_content = parser.prepare_lazy_data( + ["$a$b", "$c", {"c": "$d"}], + {}, + {"a", "b", "c", "d"} + ) + self.assertEqual( + parser.extract_variables(prepared_content), + {"a", "b", "c", "d"} ) def test_extract_functions(self): self.assertEqual( - parser.extract_functions("${func()}"), - ["func()"] + parser.regex_findall_functions("${func()}"), + [('func', '')] ) self.assertEqual( - parser.extract_functions("${func(5)}"), - ["func(5)"] + parser.regex_findall_functions("${func(5)}"), + [('func', '5')] ) self.assertEqual( - parser.extract_functions("${func(a=1, b=2)}"), - ["func(a=1, b=2)"] + parser.regex_findall_functions("${func(a=1, b=2)}"), + [('func', 'a=1, b=2')] ) self.assertEqual( - parser.extract_functions("${func(1, $b, c=$x, d=4)}"), - ["func(1, $b, c=$x, d=4)"] + parser.regex_findall_functions("${func(1, $b, c=$x, d=4)}"), + [('func', '1, $b, c=$x, d=4')] ) self.assertEqual( - parser.extract_functions("/api/1000?_t=${get_timestamp()}"), - ["get_timestamp()"] + parser.regex_findall_functions("/api/1000?_t=${get_timestamp()}"), + [('get_timestamp', '')] ) self.assertEqual( - parser.extract_functions("/api/${add(1, 2)}"), - ["add(1, 2)"] + parser.regex_findall_functions("/api/${add(1, 2)}"), + [('add', '1, 2')] ) self.assertEqual( - parser.extract_functions("/api/${add(1, 2)}?_t=${get_timestamp()}"), - ["add(1, 2)", "get_timestamp()"] + parser.regex_findall_functions("/api/${add(1, 2)}?_t=${get_timestamp()}"), + [('add', '1, 2'), ('get_timestamp', '')] ) self.assertEqual( - parser.extract_functions("abc${func(1, 2, a=3, b=4)}def"), - ["func(1, 2, a=3, b=4)"] + parser.regex_findall_functions("abc${func(1, 2, a=3, b=4)}def"), + [('func', '1, 2, a=3, b=4')] ) def test_parse_data(self): @@ -172,7 +205,7 @@ class TestParser(unittest.TestCase): functions_mapping = { "add_one": lambda x: x + 1 } - result = parser.parse_data(content, variables_mapping, functions_mapping) + result = parser.eval_lazy_data(content, variables_mapping, functions_mapping) self.assertEqual("/api/users/1000", result["request"]["url"]) self.assertEqual("abc123", result["request"]["headers"]["token"]) self.assertEqual("POST", result["request"]["method"]) @@ -182,7 +215,7 @@ class TestParser(unittest.TestCase): self.assertEqual("", result["request"]["data"]["empty_str"]) self.assertEqual("abc4def", result["request"]["data"]["value"]) - def test_parse_data_variables(self): + def test_eval_lazy_data(self): variables_mapping = { "var_1": "abc", "var_2": "def", @@ -192,66 +225,150 @@ class TestParser(unittest.TestCase): "var_6": None } self.assertEqual( - parser.parse_data("$var_1", variables_mapping), + parser.eval_lazy_data("$var_1", variables_mapping=variables_mapping), "abc" ) self.assertEqual( - parser.parse_data("var_1", variables_mapping), + parser.eval_lazy_data("var_1", variables_mapping=variables_mapping), "var_1" ) self.assertEqual( - parser.parse_data("$var_1#XYZ", variables_mapping), + parser.eval_lazy_data("$var_1#XYZ", variables_mapping=variables_mapping), "abc#XYZ" ) self.assertEqual( - parser.parse_data("/$var_1/$var_2/var3", variables_mapping), + parser.eval_lazy_data("/$var_1/$var_2/var3", variables_mapping=variables_mapping), "/abc/def/var3" ) self.assertEqual( - parser.parse_data("/$var_1/$var_2/$var_1", variables_mapping), + parser.eval_lazy_data("/$var_1/$var_2/$var_1", variables_mapping=variables_mapping), "/abc/def/abc" ) self.assertEqual( - parser.parse_string_variables("${func($var_1, $var_2, xyz)}", variables_mapping, {}), - "${func(abc, def, xyz)}" - ) - self.assertEqual( - parser.parse_data("$var_3", variables_mapping), + parser.eval_lazy_data("$var_3", variables_mapping=variables_mapping), 123 ) self.assertEqual( - parser.parse_data("$var_4", variables_mapping), + parser.eval_lazy_data("$var_4", variables_mapping=variables_mapping), {"a": 1} ) self.assertEqual( - parser.parse_data("$var_5", variables_mapping), + parser.eval_lazy_data("$var_5", variables_mapping=variables_mapping), True ) self.assertEqual( - parser.parse_data("abc$var_5", variables_mapping), + parser.eval_lazy_data("abc$var_5", variables_mapping=variables_mapping), "abcTrue" ) self.assertEqual( - parser.parse_data("abc$var_4", variables_mapping), + parser.eval_lazy_data("abc$var_4", variables_mapping=variables_mapping), "abc{'a': 1}" ) self.assertEqual( - parser.parse_data("$var_6", variables_mapping), + parser.eval_lazy_data("$var_6", variables_mapping=variables_mapping), None ) with self.assertRaises(exceptions.VariableNotFound): - parser.parse_data("/api/$SECRET_KEY", variables_mapping) + parser.eval_lazy_data("/api/$SECRET_KEY", variables_mapping=variables_mapping) self.assertEqual( - parser.parse_data(["$var_1", "$var_2"], variables_mapping), + parser.eval_lazy_data(["$var_1", "$var_2"], variables_mapping=variables_mapping), ["abc", "def"] ) self.assertEqual( - parser.parse_data({"$var_1": "$var_2"}, variables_mapping), + parser.eval_lazy_data({"$var_1": "$var_2"}, variables_mapping=variables_mapping), {"abc": "def"} ) + def test_lazy_string(self): + variables_mapping = { + "var_1": "abc", + "var_2": "def", + "var_3": 123, + "var_4": {"a": 1}, + "var_5": True, + "var_6": None + } + check_variables_set = variables_mapping.keys() + functions_mapping = { + "func1": lambda x,y: str(x) + str(y) + } + + var = parser.LazyString("ABC$var_1", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}") + self.assertEqual(var._args, ["var_1"]) + self.assertEqual(var.to_value(variables_mapping), "ABCabc") + + var = parser.LazyString("ABC$var_1$var_3", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}{}") + self.assertEqual(var._args, ["var_1", "var_3"]) + self.assertEqual(var.to_value(variables_mapping), "ABCabc123") + + var = parser.LazyString("ABC$var_1/$var_3", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}/{}") + self.assertEqual(var._args, ["var_1", "var_3"]) + self.assertEqual(var.to_value(variables_mapping), "ABCabc/123") + + var = parser.LazyString("ABC$var_1/", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}/") + self.assertEqual(var._args, ["var_1"]) + self.assertEqual(var.to_value(variables_mapping), "ABCabc/") + + var = parser.LazyString("ABC$var_1$", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}$") + self.assertEqual(var._args, ["var_1"]) + self.assertEqual(var.to_value(variables_mapping), "ABCabc$") + + var = parser.LazyString("ABC$var_1{", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}{") + self.assertEqual(var._args, ["var_1"]) + # self.assertEqual(var.to_value(variables_mapping), "ABCabc{") + + var = parser.LazyString("ABC$$var_1{", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC${}{") + self.assertEqual(var._args, ["var_1"]) + + var = parser.LazyString("ABC$var_1${", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}${") + self.assertEqual(var._args, ["var_1"]) + + var = parser.LazyString("ABC$var_1${a", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}${a") + self.assertEqual(var._args, ["var_1"]) + + var = parser.LazyString("ABC$var_1/$var_2/$var_1", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}/{}/{}") + self.assertEqual(var._args, ["var_1", "var_2", "var_1"]) + self.assertEqual(var.to_value(variables_mapping), "ABCabc/def/abc") + + var = parser.LazyString("func1($var_1, $var_3)", functions_mapping, check_variables_set) + self.assertEqual(var._string, "func1({}, {})") + self.assertEqual(var._args, ["var_1", "var_3"]) + self.assertEqual(var.to_value(variables_mapping), "func1(abc, 123)") + + var = parser.LazyString("${func1($var_1, $var_3)}", functions_mapping, check_variables_set) + self.assertEqual(var._string, "{}") + self.assertIsInstance(var._args[0], parser.LazyFunction) + self.assertEqual(var.to_value(variables_mapping), "abc123") + + var = parser.LazyString("ABC${func1($var_1, $var_3)}DE", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}DE") + self.assertIsInstance(var._args[0], parser.LazyFunction) + self.assertEqual(var.to_value(variables_mapping), "ABCabc123DE") + + var = parser.LazyString("ABC${func1($var_1, $var_3)}$var_5", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}{}") + self.assertEqual(var.to_value(variables_mapping), "ABCabc123True") + + var = parser.LazyString("ABC${func1($var_1, $var_3)}DE$var_4", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}DE{}") + self.assertEqual(var.to_value(variables_mapping), "ABCabc123DE{'a': 1}") + + var = parser.LazyString("ABC$var_5${func1($var_1, $var_3)}", functions_mapping, check_variables_set) + self.assertEqual(var._string, "ABC{}{}") + self.assertEqual(var.to_value(variables_mapping), "ABCTrueabc123") + def test_parse_data_multiple_identical_variables(self): variables_mapping = { "userid": 100, @@ -259,7 +376,7 @@ class TestParser(unittest.TestCase): } content = "/users/$userid/training/$data?userId=$userid&data=$data" self.assertEqual( - parser.parse_data(content, variables_mapping), + parser.eval_lazy_data(content, variables_mapping=variables_mapping), "/users/100/training/1498?userId=100&data=1498" ) @@ -270,36 +387,35 @@ class TestParser(unittest.TestCase): } content = "/users/$user/$userid/$data?userId=$userid&data=$data" self.assertEqual( - parser.parse_data(content, variables_mapping), + parser.eval_lazy_data(content, variables_mapping=variables_mapping), "/users/100/1000/1498?userId=1000&data=1498" ) def test_parse_data_functions(self): import random, string functions_mapping = { - "gen_random_string": lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) \ - for _ in range(str_len)) + "gen_random_string": gen_random_string } - result = parser.parse_data("${gen_random_string(5)}", functions_mapping=functions_mapping) + result = parser.eval_lazy_data("${gen_random_string(5)}", functions_mapping=functions_mapping) self.assertEqual(len(result), 5) add_two_nums = lambda a, b=1: a + b functions_mapping["add_two_nums"] = add_two_nums self.assertEqual( - parser.parse_data("${add_two_nums(1)}", functions_mapping=functions_mapping), + parser.eval_lazy_data("${add_two_nums(1)}", functions_mapping=functions_mapping), 2 ) self.assertEqual( - parser.parse_data("${add_two_nums(1, 2)}", functions_mapping=functions_mapping), + parser.eval_lazy_data("${add_two_nums(1, 2)}", functions_mapping=functions_mapping), 3 ) self.assertEqual( - parser.parse_data("/api/${add_two_nums(1, 2)}", functions_mapping=functions_mapping), + parser.eval_lazy_data("/api/${add_two_nums(1, 2)}", functions_mapping=functions_mapping), "/api/3" ) with self.assertRaises(exceptions.FunctionNotFound): - parser.parse_data("/api/${gen_md5(abc)}") + parser.eval_lazy_data("/api/${gen_md5(abc)}", functions_mapping=functions_mapping) def test_parse_data_testcase(self): variables = { @@ -323,7 +439,11 @@ class TestParser(unittest.TestCase): }, "body": "$data" } - parsed_testcase = parser.parse_data(testcase_template, variables, functions) + parsed_testcase = parser.eval_lazy_data( + testcase_template, + variables_mapping=variables, + functions_mapping=functions + ) self.assertEqual( parsed_testcase["url"], "http://127.0.0.1:5000/api/users/1000/3" @@ -345,25 +465,133 @@ class TestParser(unittest.TestCase): 3 ) - def test_substitute_variables(self): - content = { - 'request': { - 'url': '/api/users/$uid?id=$id', - 'headers': {'token': '$token'} - } + def test_parse_variables_mapping(self): + variables = { + "varA": "123$varB", + "varB": "456$varC", + "varC": "${sum_two($a, $b)}", + "a": 1, + "b": 2 } - variables_mapping = {"$uid": 1000, "$id": 2} - substituted_data = parser.substitute_variables(content, variables_mapping) - self.assertEqual(substituted_data["request"]["url"], "/api/users/1000?id=2") - self.assertEqual(substituted_data["request"]["headers"], {'token': '$token'}) + functions = { + "sum_two": sum_two + } + prepared_variables = parser.prepare_lazy_data(variables, functions, variables.keys()) + parsed_variables = parser.parse_variables_mapping(prepared_variables) + self.assertEqual(parsed_variables["varA"], "1234563") + self.assertEqual(parsed_variables["varB"], "4563") + self.assertEqual(parsed_variables["varC"], 3) + + def test_parse_variables_mapping_fix_duplicate_function_call(self): + # fix duplicate function calling + variables = { + "varA": "$varB", + "varB": "${gen_random_string(5)}" + } + functions = { + "gen_random_string": gen_random_string + } + prepared_variables = parser.prepare_lazy_data(variables, functions, variables.keys()) + parsed_variables = parser.parse_variables_mapping(prepared_variables) + self.assertEqual(parsed_variables["varA"], parsed_variables["varB"]) + + def test_parse_variables_mapping_dead_circle(self): + variables = { + "varA": "$varB", + "varB": "123$varC" + } + check_variables_set = {"varA", "varB", "varC"} + prepared_variables = parser.prepare_lazy_data(variables, {}, check_variables_set) + with self.assertRaises(exceptions.VariableNotFound): + parser.parse_variables_mapping(prepared_variables) + + def test_parse_variables_mapping_not_found(self): + variables = { + "varA": "123$varB", + "varB": "456$varC", + "varC": "${sum_two($a, $b)}", + "b": 2 + } + functions = { + "sum_two": sum_two + } + with self.assertRaises(exceptions.VariableNotFound): + parser.prepare_lazy_data(variables, functions, variables.keys()) + + def test_parse_variables_mapping_ref_self(self): + variables = { + "varC": "${sum_two($a, $b)}", + "a": 1, + "b": 2, + "token": "$token" + } + functions = { + "sum_two": sum_two + } + prepared_variables = parser.prepare_lazy_data(variables, functions, variables.keys()) + with self.assertRaises(exceptions.VariableNotFound): + parser.parse_variables_mapping(prepared_variables) + + def test_parse_variables_mapping_2(self): + variables = { + "host2": "https://httprunner.org", + "num3": "${sum_two($num2, 4)}", + "num2": "${sum_two($num1, 3)}", + "num1": "${sum_two(1, 2)}" + } + functions = { + "sum_two": sum_two + } + prepared_variables = parser.prepare_lazy_data(variables, functions, variables.keys()) + parsed_testcase = parser.parse_variables_mapping(prepared_variables) + self.assertEqual(parsed_testcase["num3"], 10) + self.assertEqual(parsed_testcase["num2"], 6) + self.assertEqual(parsed_testcase["num1"], 3) + + def test_prepare_lazy_data(self): + variables = { + "host": "https://httprunner.org", + "num4": "${sum_two($num0, 5)}", + "num3": "${sum_two($num2, 4)}", + "num2": "${sum_two($num1, 3)}", + "num1": "${sum_two(1, 2)}", + "num0": 0 + } + functions = { + "sum_two": sum_two + } + parser.prepare_lazy_data( + variables, + functions, + variables.keys() + ) + + def test_prepare_lazy_data_not_found(self): + variables = { + "host": "https://httprunner.org", + "num4": "${sum_two($num0, 5)}", + "num3": "${sum_two($num2, 4)}", + "num2": "${sum_two($num1, 3)}", + "num1": "${sum_two(1, 2)}" + } + functions = { + "sum_two": sum_two + } + with self.assertRaises(exceptions.VariableNotFound): + parser.prepare_lazy_data( + variables, + functions, + variables.keys() + ) + + +class TestParser(unittest.TestCase): def test_parse_parameters_raw_list(self): parameters = [ {"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]}, {"username-password": [("user1", "111111"), ["test2", "222222"]]} ] - variables_mapping = {} - functions_mapping = {} cartesian_product_parameters = parser.parse_parameters(parameters) self.assertEqual( len(cartesian_product_parameters), @@ -441,20 +669,18 @@ class TestParser(unittest.TestCase): testcases = tests_mapping["testcases"] self.assertEqual( testcases[0]["config"]["variables"]["var_c"], - "${sum_two(1, 2)}" + "${sum_two($var_a, $var_b)}" ) self.assertEqual( testcases[0]["config"]["variables"]["PROJECT_KEY"], "${ENV(PROJECT_KEY)}" ) - parsed_tests_mapping = parser.parse_tests(tests_mapping) - parsed_testcases = parsed_tests_mapping["testcases"] + parsed_testcases = parser.parse_tests(tests_mapping) self.assertIsInstance(parsed_testcases, list) test_dict1 = parsed_testcases[0]["teststeps"][0] - self.assertEqual(test_dict1["variables"]["var_c"], 3) - self.assertEqual(test_dict1["variables"]["PROJECT_KEY"], "ABCDEFGH") - self.assertEqual(test_dict1["variables"]["var_d"], test_dict1["variables"]["var_e"]) - self.assertEqual(parsed_testcases[0]["config"]["name"], '1230') + self.assertEqual(test_dict1["variables"]["var_c"].raw_string, "${sum_two($var_a, $var_b)}") + self.assertEqual(test_dict1["variables"]["PROJECT_KEY"].raw_string, "${ENV(PROJECT_KEY)}") + self.assertIsInstance(parsed_testcases[0]["config"]["name"], parser.LazyString) def test_parse_tests_override_variables(self): tests_mapping = { @@ -480,10 +706,10 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict1_variables = parsed_tests_mapping["testcases"][0]["teststeps"][0]["variables"] + parsed_testcases = parser.parse_tests(tests_mapping) + test_dict1_variables = parsed_testcases[0]["teststeps"][0]["variables"] self.assertEqual(test_dict1_variables["creator"], "user_test_001") - self.assertEqual(test_dict1_variables["username"], "user_test_001") + self.assertEqual(test_dict1_variables["username"].raw_string, "$creator") def test_parse_tests_base_url_priority(self): """ base_url & verify: priority test_dict > config @@ -509,9 +735,9 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0] - self.assertEqual(test_dict["request"]["url"], "https://httprunner.org/api1") + parsed_testcases = parser.parse_tests(tests_mapping) + test_dict = parsed_testcases[0]["teststeps"][0] + self.assertEqual(test_dict["request"]["url"], "/api1") self.assertEqual(test_dict["request"]["verify"], True) def test_parse_tests_base_url_path_with_variable(self): @@ -537,9 +763,11 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0] - self.assertEqual(test_dict["request"]["url"], "https://httprunner.org/api1") + parsed_testcases = parser.parse_tests(tests_mapping) + test_dict = parsed_testcases[0]["teststeps"][0] + self.assertEqual(test_dict["variables"]["host2"], "https://httprunner.org") + parsed_test_dict = parser.parse_lazy_data(test_dict, test_dict["variables"]) + self.assertEqual(parsed_test_dict["request"]["url"], "https://httprunner.org/api1") def test_parse_tests_base_url_test_dict(self): tests_mapping = { @@ -565,54 +793,12 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0] - self.assertEqual(test_dict["request"]["url"], "https://httprunner.org/api1") - - def test_parse_data_with_variables(self): - variables = { - "host2": "https://httprunner.org", - "num3": "${sum_two($num2, 4)}", - "num2": "${sum_two($num1, 3)}", - "num1": "${sum_two(1, 2)}" - } - from tests.debugtalk import sum_two - functions = { - "sum_two": sum_two - } - parsed_testcase = parser.parse_data(variables, variables, functions) - self.assertEqual(parsed_testcase["num3"], 10) - self.assertEqual(parsed_testcase["num2"], 6) - self.assertEqual(parsed_testcase["num1"], 3) - - def test_parse_data_with_variables_not_found(self): - variables = { - "host": "https://httprunner.org", - "num4": "${sum_two($num0, 5)}", - "num3": "${sum_two($num2, 4)}", - "num2": "${sum_two($num1, 3)}", - "num1": "${sum_two(1, 2)}" - } - from tests.debugtalk import sum_two - functions = { - "sum_two": sum_two - } - with self.assertRaises(exceptions.VariableNotFound): - parser.parse_data(variables, variables, functions) - - parsed_testcase = parser.parse_data( - variables, - variables, - functions, - raise_if_variable_not_found=False - ) - self.assertEqual(parsed_testcase["num3"], 10) - self.assertEqual(parsed_testcase["num2"], 6) - self.assertEqual(parsed_testcase["num1"], 3) - self.assertEqual(parsed_testcase["num4"], "${sum_two($num0, 5)}") + parsed_testcases = parser.parse_tests(tests_mapping) + test_dict = parsed_testcases[0]["teststeps"][0] + parsed_test_dict = parser.parse_lazy_data(test_dict, test_dict["variables"]) + self.assertEqual(parsed_test_dict["base_url"], "https://httprunner.org") def test_parse_tests_variable_with_function(self): - from tests.debugtalk import sum_two, gen_random_string tests_mapping = { "project_mapping": { "functions": { @@ -652,18 +838,20 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0] - self.assertEqual(test_dict["variables"]["num3"], 10) - self.assertEqual(test_dict["variables"]["num2"], 6) - self.assertEqual(test_dict["variables"]["str1"], test_dict["variables"]["str2"]) + parsed_testcases = parser.parse_tests(tests_mapping) + test_dict = parsed_testcases[0]["teststeps"][0] + variables = parser.parse_variables_mapping(test_dict["variables"]) + self.assertEqual(variables["num3"], 10) + self.assertEqual(variables["num2"], 6) + parsed_test_dict = parser.parse_lazy_data(test_dict, variables) + self.assertEqual(parsed_test_dict["base_url"], "https://httprunner.org") self.assertEqual( - test_dict["request"]["url"], - "https://httprunner.org/api1/?num1=3&num2=6&num3=10" + parsed_test_dict["request"]["url"], + "/api1/?num1=3&num2=6&num3=10" ) + self.assertEqual(variables["str1"], variables["str2"]) def test_parse_tests_variable_not_found(self): - from tests.debugtalk import sum_two tests_mapping = { "project_mapping": { "functions": { @@ -699,15 +887,8 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0] - self.assertEqual(test_dict["variables"]["num3"], 10) - self.assertEqual(test_dict["variables"]["num2"], 6) - self.assertEqual(test_dict["variables"]["num4"], "${sum_two($num0, 5)}") - self.assertEqual( - test_dict["request"]["url"], - "https://httprunner.org/api1/?num1=3&num2=6&num3=10&num4=${sum_two($num0, 5)}" - ) + with self.assertRaises(exceptions.VariableNotFound): + parser.parse_tests(tests_mapping) def test_parse_tests_base_url_teststep_empty(self): """ base_url & verify: priority test_dict > config @@ -733,9 +914,9 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0] - self.assertEqual(test_dict["request"]["url"], "https://debugtalk.com/api1") + parsed_testcases = parser.parse_tests(tests_mapping) + test_dict = parsed_testcases[0]["teststeps"][0] + self.assertEqual(str(test_dict["base_url"]), 'LazyString($host)') self.assertEqual(test_dict["request"]["verify"], True) def test_parse_tests_verify_config_set(self): @@ -758,8 +939,8 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0] + parsed_testcases = parser.parse_tests(tests_mapping) + test_dict = parsed_testcases[0]["teststeps"][0] self.assertEqual(test_dict["request"]["verify"], False) def test_parse_tests_verify_config_unset(self): @@ -781,8 +962,8 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0] + parsed_testcases = parser.parse_tests(tests_mapping) + test_dict = parsed_testcases[0]["teststeps"][0] self.assertEqual(test_dict["request"]["verify"], True) def test_parse_tests_verify_step_set_false(self): @@ -805,8 +986,8 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0] + parsed_testcases = parser.parse_tests(tests_mapping) + test_dict = parsed_testcases[0]["teststeps"][0] self.assertEqual(test_dict["request"]["verify"], False) def test_parse_tests_verify_nested_testcase_unset(self): @@ -840,8 +1021,8 @@ class TestParser(unittest.TestCase): } ] } - parsed_tests_mapping = parser.parse_tests(tests_mapping) - test_dict = parsed_tests_mapping["testcases"][0]["teststeps"][0] + parsed_testcases = parser.parse_tests(tests_mapping) + test_dict = parsed_testcases[0]["teststeps"][0] self.assertEqual(test_dict["teststeps"][0]["request"]["verify"], False) def test_parse_environ(self): @@ -851,7 +1032,7 @@ class TestParser(unittest.TestCase): {"PROJECT_KEY": "${ENV(PROJECT_KEY)}"} ] } - result = parser.parse_data(content) + result = parser.eval_lazy_data(content) content = { "variables": [ @@ -859,7 +1040,7 @@ class TestParser(unittest.TestCase): ] } with self.assertRaises(exceptions.ParamsError): - parser.parse_data(content) + parser.eval_lazy_data(content) content = { "variables": [ @@ -867,7 +1048,7 @@ class TestParser(unittest.TestCase): ] } with self.assertRaises(exceptions.ParamsError): - parser.parse_data(content) + parser.eval_lazy_data(content) def test_extend_with_api(self): loader.load_project_tests(os.path.join(os.getcwd(), "tests")) @@ -888,18 +1069,18 @@ class TestParser(unittest.TestCase): 'url': '/api/get-token', 'method': 'POST', 'headers': {'user_agent': '$user_agent', 'device_sn': '$device_sn', 'os_platform': '$os_platform', 'app_version': '$app_version'}, - 'json': {'sign': '${get_sign($user_agent, $device_sn, $os_platform, $app_version)}'} + 'json': {'sign': '${get_sign($device_sn, $os_platform, $app_version)}'} }, 'validate': [ - {'eq': ['status_code', 201]}, - {'len_eq': ['content.token', 32]} + {"check": "status_code", "comparator": "equals", "expect": 201}, + {"check": "content.token", "comparator": "length_equals", "expect": 32} ] } - extended_block = parser._extend_with_api(test_block, api_def_dict) - self.assertEqual(extended_block["base_url"], "https://debugtalk.com") - self.assertEqual(extended_block["name"], "override block") - self.assertEqual({'var': 123}, extended_block["variables"]) - self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, extended_block["validate"]) - self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, extended_block["validate"]) - self.assertEqual(extended_block["times"], 3) + parser._extend_with_api(test_block, api_def_dict) + self.assertEqual(test_block["base_url"], "https://debugtalk.com") + self.assertEqual(test_block["name"], "override block") + self.assertEqual({'var': 123}, test_block["variables"]) + self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'equals'}, test_block["validate"]) + self.assertIn({'check': 'content.token', 'comparator': 'length_equals', 'expect': 32}, test_block["validate"]) + self.assertEqual(test_block["times"], 3) diff --git a/tests/test_runner.py b/tests/test_runner.py index 936c879b..4b684c1d 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,8 +1,7 @@ import os import time -from httprunner import exceptions, loader, runner -from httprunner.utils import deep_update_dict +from httprunner import loader, parser, runner from tests.api_server import HTTPBIN_SERVER from tests.base import ApiServerUnittest @@ -19,7 +18,7 @@ class TestRunner(ApiServerUnittest): "base_url": "http://127.0.0.1", "verify": False } - self.test_runner = runner.Runner(config, self.debugtalk_functions) + self.test_runner = runner.Runner(config) self.reset_all() def reset_all(self): @@ -36,204 +35,253 @@ class TestRunner(ApiServerUnittest): ] for testcase_file_path in testcase_file_path_list: - testcases = loader.load_file(testcase_file_path) - - config_dict = {} - test_runner = runner.Runner(config_dict, self.debugtalk_functions) - - test = testcases[0]["test"] - test_runner.run_test(test) - - test = testcases[1]["test"] - test_runner.run_test(test) - - test = testcases[2]["test"] - test_runner.run_test(test) - - def test_run_single_testcase_fail(self): - test = { - "name": "get token", - "request": { - "url": "http://127.0.0.1:5000/api/get-token", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3", - "device_sn": "HZfFBh6tU59EdXJ", - "os_platform": "ios", - "app_version": "2.8.6" - }, - "json": { - "sign": "f1219719911caae89ccc301679857ebfda115ca2" - } - }, - "validate": [ - {"check": "status_code", "expect": 205}, - {"check": "content.token", "comparator": "len_eq", "expect": 19} - ] - } - - with self.assertRaises(exceptions.ValidationFailure): - self.test_runner.run_test(test) + tests_mapping = loader.load_tests(testcase_file_path) + parsed_testcases = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_testcases[0] + test_runner = runner.Runner(parsed_testcase["config"]) + test_runner.run_test(parsed_testcase["teststeps"][0]) + test_runner.run_test(parsed_testcase["teststeps"][1]) + test_runner.run_test(parsed_testcase["teststeps"][2]) def test_run_testcase_with_hooks(self): start_time = time.time() - config_dict = { - "name": "basic test with httpbin", - "base_url": HTTPBIN_SERVER, - "setup_hooks": [ - "${sleep_N_secs(0.5)}" - "${hook_print(setup)}" - ], - "teardown_hooks": [ - "${sleep_N_secs(1)}", - "${hook_print(teardown)}" - ] - } - test = { - "name": "get token", - "request": { - "url": "http://127.0.0.1:5000/api/get-token", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3", - "device_sn": "HZfFBh6tU59EdXJ", - "os_platform": "ios", - "app_version": "2.8.6" + testcases = [ + { + "config": { + "name": "basic test with httpbin", + "base_url": HTTPBIN_SERVER, + "setup_hooks": [ + "${sleep_N_secs(0.5)}", + "${hook_print(setup)}" + ], + "teardown_hooks": [ + "${sleep_N_secs(1)}", + "${hook_print(teardown)}" + ] }, - "json": { - "sign": "f1219719911caae89ccc301679857ebfda115ca2" - } + "teststeps": [ + { + "name": "get token", + "request": { + "url": "http://127.0.0.1:5000/api/get-token", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3", + "device_sn": "HZfFBh6tU59EdXJ", + "os_platform": "ios", + "app_version": "2.8.6" + }, + "json": { + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" + } + }, + "validate": [ + {"check": "status_code", "expect": 200} + ] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "validate": [ - {"check": "status_code", "expect": 200} - ] + "testcases": testcases } - test_runner = runner.Runner(config_dict, self.debugtalk_functions) + parsed_testcases = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_testcases[0] + test_runner = runner.Runner(parsed_testcase["config"]) end_time = time.time() # check if testcase setup hook executed self.assertGreater(end_time - start_time, 0.5) start_time = time.time() - test_runner.run_test(test) - test_runner.run_test(test) + test_runner.run_test(parsed_testcase["teststeps"][0]) end_time = time.time() # testcase teardown hook has not been executed now self.assertLess(end_time - start_time, 1) def test_run_testcase_with_hooks_assignment(self): - config_dict = { - "name": "basic test with httpbin", - "base_url": HTTPBIN_SERVER - } - test = { - "name": "modify request headers", - "request": { - "url": "/anything", - "method": "POST", - "headers": { - "user_agent": "iOS/10.3", - "os_platform": "ios" + testcases = [ + { + "config": { + "name": "basic test with httpbin", + "base_url": HTTPBIN_SERVER }, - "data": "a=1&b=2" + "teststeps": [ + { + "name": "modify request headers", + "base_url": HTTPBIN_SERVER, + "request": { + "url": "/anything", + "method": "POST", + "headers": { + "user_agent": "iOS/10.3", + "os_platform": "ios" + }, + "data": "a=1&b=2" + }, + "setup_hooks": [ + {"total": "${sum_two(1, 5)}"} + ], + "validate": [ + {"check": "status_code", "expect": 200} + ] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "setup_hooks": [ - {"total": "${sum_two(1, 5)}"} - ], - "validate": [ - {"check": "status_code", "expect": 200} - ] + "testcases": testcases } - test_runner = runner.Runner(config_dict, self.debugtalk_functions) - test_runner.run_test(test) + parsed_testcases = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_testcases[0] + test_runner = runner.Runner(parsed_testcase["config"]) + test_runner.run_test(parsed_testcase["teststeps"][0]) test_variables_mapping = test_runner.session_context.test_variables_mapping self.assertEqual(test_variables_mapping["total"], 6) self.assertEqual(test_variables_mapping["request"]["data"], "a=1&b=2") def test_run_testcase_with_hooks_modify_request(self): - config_dict = { - "name": "basic test with httpbin", - "base_url": HTTPBIN_SERVER - } - test = { - "name": "modify request headers", - "request": { - "url": "/anything", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3" + testcases = [ + { + "config": { + "name": "basic test with httpbin", + "base_url": HTTPBIN_SERVER }, - "json": { - "os_platform": "ios", - "sign": "f1219719911caae89ccc301679857ebfda115ca2" - } + "teststeps": [ + { + "name": "modify request headers", + "base_url": HTTPBIN_SERVER, + "request": { + "url": "/anything", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3" + }, + "json": { + "os_platform": "ios", + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" + } + }, + "setup_hooks": [ + "${modify_request_json($request, android)}" + ], + "validate": [ + {"check": "status_code", "expect": 200}, + {"check": "content.json.os_platform", "expect": "android"} + ] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "setup_hooks": [ - "${modify_request_json($request, android)}" - ], - "validate": [ - {"check": "status_code", "expect": 200}, - {"check": "content.json.os_platform", "expect": "android"} - ] + "testcases": testcases } - test_runner = runner.Runner(config_dict, self.debugtalk_functions) - test_runner.run_test(test) + parsed_testcases = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_testcases[0] + test_runner = runner.Runner(parsed_testcase["config"]) + test_runner.run_test(parsed_testcase["teststeps"][0]) def test_run_testcase_with_teardown_hooks_success(self): - test = { - "name": "get token", - "request": { - "url": "http://127.0.0.1:5000/api/get-token", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3", - "device_sn": "HZfFBh6tU59EdXJ", - "os_platform": "ios", - "app_version": "2.8.6" + testcases = [ + { + "config": { + "name": "basic test with httpbin" }, - "json": { - "sign": "f1219719911caae89ccc301679857ebfda115ca2" - } + "teststeps": [ + { + "name": "get token", + "request": { + "url": "http://127.0.0.1:5000/api/get-token", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3", + "device_sn": "HZfFBh6tU59EdXJ", + "os_platform": "ios", + "app_version": "2.8.6" + }, + "json": { + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" + } + }, + "validate": [ + {"check": "status_code", "expect": 200} + ], + "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "validate": [ - {"check": "status_code", "expect": 200} - ], - "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] + "testcases": testcases } + parsed_testcases = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_testcases[0] + test_runner = runner.Runner(parsed_testcase["config"]) + start_time = time.time() - self.test_runner.run_test(test) + test_runner.run_test(parsed_testcase["teststeps"][0]) end_time = time.time() # check if teardown function executed self.assertLess(end_time - start_time, 0.5) def test_run_testcase_with_teardown_hooks_fail(self): - test = { - "name": "get token", - "request": { - "url": "http://127.0.0.1:5000/api/get-token2", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3", - "device_sn": "HZfFBh6tU59EdXJ", - "os_platform": "ios", - "app_version": "2.8.6" + testcases = [ + { + "config": { + "name": "basic test with httpbin" }, - "json": { - "sign": "f1219719911caae89ccc301679857ebfda115ca2" - } + "teststeps": [ + { + "name": "get token", + "request": { + "url": "http://127.0.0.1:5000/api/get-token2", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3", + "device_sn": "HZfFBh6tU59EdXJ", + "os_platform": "ios", + "app_version": "2.8.6" + }, + "json": { + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" + } + }, + "validate": [ + {"check": "status_code", "expect": 404} + ], + "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "validate": [ - {"check": "status_code", "expect": 404} - ], - "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] + "testcases": testcases } + parsed_testcases = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_testcases[0] + test_runner = runner.Runner(parsed_testcase["config"]) + start_time = time.time() - self.test_runner.run_test(test) + test_runner.run_test(parsed_testcase["teststeps"][0]) end_time = time.time() # check if teardown function executed self.assertGreater(end_time - start_time, 2) @@ -241,34 +289,51 @@ class TestRunner(ApiServerUnittest): def test_bugfix_type_match(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/bugfix_type_match.yml') - testcases = loader.load_file(testcase_file_path) - - test = testcases[1]["test"] - self.test_runner.run_test(test) + tests_mapping = loader.load_tests(testcase_file_path) + parsed_testcases = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_testcases[0] + test_runner = runner.Runner(parsed_testcase["config"]) + test_runner.run_test(parsed_testcase["teststeps"][0]) def test_run_validate_elapsed(self): - test = { - "name": "get token", - "request": { - "url": "http://127.0.0.1:5000/api/get-token", - "method": "POST", - "headers": { - "content-type": "application/json", - "user_agent": "iOS/10.3", - "device_sn": "HZfFBh6tU59EdXJ", - "os_platform": "ios", - "app_version": "2.8.6" - }, - "json": { - "sign": "f1219719911caae89ccc301679857ebfda115ca2" - } + testcases = [ + { + "config": {}, + "teststeps": [ + { + "name": "get token", + "request": { + "url": "http://127.0.0.1:5000/api/get-token", + "method": "POST", + "headers": { + "content-type": "application/json", + "user_agent": "iOS/10.3", + "device_sn": "HZfFBh6tU59EdXJ", + "os_platform": "ios", + "app_version": "2.8.6" + }, + "json": { + "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" + } + }, + "validate": [ + {"check": "status_code", "expect": 200}, + {"check": "elapsed.seconds", "comparator": "lt", "expect": 1}, + {"check": "elapsed.days", "comparator": "eq", "expect": 0}, + {"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000}, + {"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1} + ] + } + ] + } + ] + tests_mapping = { + "project_mapping": { + "functions": self.debugtalk_functions }, - "validate": [ - {"check": "status_code", "expect": 200}, - {"check": "elapsed.seconds", "comparator": "lt", "expect": 1}, - {"check": "elapsed.days", "comparator": "eq", "expect": 0}, - {"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000}, - {"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1} - ] + "testcases": testcases } - self.test_runner.run_test(test) + parsed_testcases = parser.parse_tests(tests_mapping) + parsed_testcase = parsed_testcases[0] + test_runner = runner.Runner(parsed_testcase["config"]) + test_runner.run_test(parsed_testcase["teststeps"][0]) \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index 354a906a..9a804b2a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -61,35 +61,6 @@ class TestUtils(ApiServerUnittest): result = utils.query_json(json_content, query) self.assertEqual(result, "L") - def test_get_uniform_comparator(self): - self.assertEqual(utils.get_uniform_comparator("eq"), "equals") - self.assertEqual(utils.get_uniform_comparator("=="), "equals") - self.assertEqual(utils.get_uniform_comparator("lt"), "less_than") - self.assertEqual(utils.get_uniform_comparator("le"), "less_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("gt"), "greater_than") - self.assertEqual(utils.get_uniform_comparator("ge"), "greater_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("ne"), "not_equals") - - self.assertEqual(utils.get_uniform_comparator("str_eq"), "string_equals") - self.assertEqual(utils.get_uniform_comparator("len_eq"), "length_equals") - self.assertEqual(utils.get_uniform_comparator("count_eq"), "length_equals") - - self.assertEqual(utils.get_uniform_comparator("len_gt"), "length_greater_than") - self.assertEqual(utils.get_uniform_comparator("count_gt"), "length_greater_than") - self.assertEqual(utils.get_uniform_comparator("count_greater_than"), "length_greater_than") - - self.assertEqual(utils.get_uniform_comparator("len_ge"), "length_greater_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("count_ge"), "length_greater_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("count_greater_than_or_equals"), "length_greater_than_or_equals") - - self.assertEqual(utils.get_uniform_comparator("len_lt"), "length_less_than") - self.assertEqual(utils.get_uniform_comparator("count_lt"), "length_less_than") - self.assertEqual(utils.get_uniform_comparator("count_less_than"), "length_less_than") - - self.assertEqual(utils.get_uniform_comparator("len_le"), "length_less_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("count_le"), "length_less_than_or_equals") - self.assertEqual(utils.get_uniform_comparator("count_less_than_or_equals"), "length_less_than_or_equals") - def current_validators(self): from httprunner import built_in functions_mapping = loader.load_module_functions(built_in) @@ -205,61 +176,6 @@ class TestUtils(ApiServerUnittest): self.assertIsInstance(ordered_dict, dict) self.assertIn("a", ordered_dict) - def test_extend_validators(self): - def_validators = [ - {'eq': ['v1', 200]}, - {"check": "s2", "expect": 16, "comparator": "len_eq"} - ] - current_validators = [ - {"check": "v1", "expect": 201}, - {'len_eq': ['s3', 12]} - ] - def_validators = [ - parser.parse_validator(validator) - for validator in def_validators - ] - ref_validators = [ - parser.parse_validator(validator) - for validator in current_validators - ] - - extended_validators = utils.extend_validators(def_validators, ref_validators) - self.assertIn( - {"check": "v1", "expect": 201, "comparator": "eq"}, - extended_validators - ) - self.assertIn( - {"check": "s2", "expect": 16, "comparator": "len_eq"}, - extended_validators - ) - self.assertIn( - {"check": "s3", "expect": 12, "comparator": "len_eq"}, - extended_validators - ) - - def test_extend_validators_with_dict(self): - def_validators = [ - {'eq': ["a", {"v": 1}]}, - {'eq': [{"b": 1}, 200]} - ] - current_validators = [ - {'len_eq': ['s3', 12]}, - {'eq': [{"b": 1}, 201]} - ] - def_validators = [ - parser.parse_validator(validator) - for validator in def_validators - ] - ref_validators = [ - parser.parse_validator(validator) - for validator in current_validators - ] - - extended_validators = utils.extend_validators(def_validators, ref_validators) - self.assertEqual(len(extended_validators), 3) - self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'eq'}, extended_validators) - self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'eq'}, extended_validators) - def test_extend_variables(self): raw_variables = [{"var1": "val1"}, {"var2": "val2"}] override_variables = [{"var1": "val111"}, {"var3": "val3"}] diff --git a/tests/test_validator.py b/tests/test_validator.py index 64e97b2d..63a8cc5a 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -74,3 +74,101 @@ class TestValidator(unittest.TestCase): func = lambda x: x + 1 self.assertTrue(validator.is_function(func)) self.assertTrue(validator.is_function(validator.is_testcase)) + + def test_get_uniform_comparator(self): + self.assertEqual(validator.get_uniform_comparator("eq"), "equals") + self.assertEqual(validator.get_uniform_comparator("=="), "equals") + self.assertEqual(validator.get_uniform_comparator("lt"), "less_than") + self.assertEqual(validator.get_uniform_comparator("le"), "less_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("gt"), "greater_than") + self.assertEqual(validator.get_uniform_comparator("ge"), "greater_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("ne"), "not_equals") + + self.assertEqual(validator.get_uniform_comparator("str_eq"), "string_equals") + self.assertEqual(validator.get_uniform_comparator("len_eq"), "length_equals") + self.assertEqual(validator.get_uniform_comparator("count_eq"), "length_equals") + + self.assertEqual(validator.get_uniform_comparator("len_gt"), "length_greater_than") + self.assertEqual(validator.get_uniform_comparator("count_gt"), "length_greater_than") + self.assertEqual(validator.get_uniform_comparator("count_greater_than"), "length_greater_than") + + self.assertEqual(validator.get_uniform_comparator("len_ge"), "length_greater_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("count_ge"), "length_greater_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("count_greater_than_or_equals"), "length_greater_than_or_equals") + + self.assertEqual(validator.get_uniform_comparator("len_lt"), "length_less_than") + self.assertEqual(validator.get_uniform_comparator("count_lt"), "length_less_than") + self.assertEqual(validator.get_uniform_comparator("count_less_than"), "length_less_than") + + self.assertEqual(validator.get_uniform_comparator("len_le"), "length_less_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("count_le"), "length_less_than_or_equals") + self.assertEqual(validator.get_uniform_comparator("count_less_than_or_equals"), "length_less_than_or_equals") + + def test_parse_validator(self): + _validator = {"check": "status_code", "comparator": "eq", "expect": 201} + self.assertEqual( + validator.uniform_validator(_validator), + {"check": "status_code", "comparator": "equals", "expect": 201} + ) + + _validator = {'eq': ['status_code', 201]} + self.assertEqual( + validator.uniform_validator(_validator), + {"check": "status_code", "comparator": "equals", "expect": 201} + ) + + + def test_extend_validators(self): + def_validators = [ + {'eq': ['v1', 200]}, + {"check": "s2", "expect": 16, "comparator": "len_eq"} + ] + current_validators = [ + {"check": "v1", "expect": 201}, + {'len_eq': ['s3', 12]} + ] + def_validators = [ + validator.uniform_validator(_validator) + for _validator in def_validators + ] + ref_validators = [ + validator.uniform_validator(_validator) + for _validator in current_validators + ] + + extended_validators = validator.extend_validators(def_validators, ref_validators) + self.assertIn( + {"check": "v1", "expect": 201, "comparator": "equals"}, + extended_validators + ) + self.assertIn( + {"check": "s2", "expect": 16, "comparator": "length_equals"}, + extended_validators + ) + self.assertIn( + {"check": "s3", "expect": 12, "comparator": "length_equals"}, + extended_validators + ) + + def test_extend_validators_with_dict(self): + def_validators = [ + {'eq': ["a", {"v": 1}]}, + {'eq': [{"b": 1}, 200]} + ] + current_validators = [ + {'len_eq': ['s3', 12]}, + {'eq': [{"b": 1}, 201]} + ] + def_validators = [ + validator.uniform_validator(_validator) + for _validator in def_validators + ] + ref_validators = [ + validator.uniform_validator(_validator) + for _validator in current_validators + ] + + extended_validators = validator.extend_validators(def_validators, ref_validators) + self.assertEqual(len(extended_validators), 3) + self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'equals'}, extended_validators) + self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'equals'}, extended_validators) diff --git a/tests/testcases/create_user.yml b/tests/testcases/create_user.yml new file mode 100644 index 00000000..a09160cc --- /dev/null +++ b/tests/testcases/create_user.yml @@ -0,0 +1,22 @@ + +- config: + name: "create user and check result." + id: create_user + base_url: "http://127.0.0.1:5000" + variables: + uid: 9001 + device_sn: "TESTCASE_CREATE_XXX" + output: + - session_token + +- test: + name: setup and reset all (override) for $device_sn. + testcase: testcases/setup.yml + output: + - session_token + +- test: + name: create user and check result. + variables: + token: $session_token + testcase: testcases/deps/check_and_create.yml diff --git a/tests/testcases/create_and_check.yml b/tests/testcases/deps/check_and_create.yml similarity index 85% rename from tests/testcases/create_and_check.yml rename to tests/testcases/deps/check_and_create.yml index f2424f93..8a6b2d0a 100644 --- a/tests/testcases/create_and_check.yml +++ b/tests/testcases/deps/check_and_create.yml @@ -1,4 +1,3 @@ - - config: name: "create user and check result." id: create_and_check @@ -6,14 +5,6 @@ variables: uid: 9001 device_sn: "TESTCASE_CREATE_XXX" - output: - - token - -- test: - name: setup and reset all (override) for $device_sn. - testcase: testcases/setup.yml - output: - - token - test: name: make sure user $uid does not exist diff --git a/tests/testcases/setup.yml b/tests/testcases/setup.yml index 24414d9f..344ff395 100644 --- a/tests/testcases/setup.yml +++ b/tests/testcases/setup.yml @@ -9,7 +9,7 @@ base_url: "http://127.0.0.1:5000" verify: False output: - - token + - session_token - test: name: get token (setup) @@ -20,7 +20,7 @@ os_platform: 'ios' app_version: '2.8.6' extract: - - token: content.token + - session_token: content.token validate: - eq: ["status_code", 200] - len_eq: ["content.token", 16] @@ -29,4 +29,4 @@ name: reset all users api: api/reset_all.yml variables: - token: $token + token: $session_token diff --git a/tests/testsuites/create_users.yml b/tests/testsuites/create_users.yml index 8d67996c..25c567a5 100644 --- a/tests/testsuites/create_users.yml +++ b/tests/testsuites/create_users.yml @@ -8,14 +8,14 @@ config: testcases: create user 1000 and check result.: - testcase: testcases/create_and_check.yml + testcase: testcases/create_user.yml variables: uid: 1000 var_c: ${gen_random_string(5)} var_d: $var_c create user 1001 and check result.: - testcase: testcases/create_and_check.yml + testcase: testcases/create_user.yml variables: uid: 1001 var_c: ${gen_random_string(5)} diff --git a/tests/testsuites/create_users_with_parameters.yml b/tests/testsuites/create_users_with_parameters.yml index ae2cd939..cd608af7 100644 --- a/tests/testsuites/create_users_with_parameters.yml +++ b/tests/testsuites/create_users_with_parameters.yml @@ -6,7 +6,7 @@ config: testcases: create user $uid and check result for $device_sn.: - testcase: testcases/create_and_check.yml + testcase: testcases/create_user.yml variables: uid: 1000 device_sn: TESTSUITE_XXX