From e2b369b305e0fb9d9498c5c8cb95ea10459e52cf Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 18 Apr 2020 11:54:36 +0800 Subject: [PATCH 01/49] test: add testcase example, set & delete cookies --- .../set_delete_cookies.yml | 41 +++++++++++++++++++ examples/postman_echo/debugtalk.py | 5 +++ 2 files changed, 46 insertions(+) create mode 100644 examples/postman_echo/cookie_manipulation/set_delete_cookies.yml create mode 100644 examples/postman_echo/debugtalk.py diff --git a/examples/postman_echo/cookie_manipulation/set_delete_cookies.yml b/examples/postman_echo/cookie_manipulation/set_delete_cookies.yml new file mode 100644 index 00000000..f43116a9 --- /dev/null +++ b/examples/postman_echo/cookie_manipulation/set_delete_cookies.yml @@ -0,0 +1,41 @@ +config: + name: "set & delete cookies." + variables: + foo1: bar1 + foo2: bar2 + base_url: "https://postman-echo.com" + verify: False + export: ["cookie_foo1", "cookie_foo3"] + +teststeps: +- + name: set cookie foo1 & foo2 & foo3 + variables: + foo3: bar3 + request: + method: GET + url: /cookies/set + params: + foo1: bar111 + foo2: $foo2 + foo3: $foo3 + headers: + User-Agent: HttpRunner/${get_httprunner_version()} + extract: + cookie_foo1: $.cookies.foo1 + cookie_foo3: $.cookies.foo3 + validate: + - eq: ["status_code", 200] + - ne: ["$.cookies.foo3", "$foo3"] +- + name: delete cookie foo2 + request: + method: GET + url: /cookies/delete?foo2 + headers: + User-Agent: HttpRunner/${get_httprunner_version()} + validate: + - eq: ["status_code", 200] + - ne: ["$.cookies.foo1", "$foo1"] + - eq: ["$.cookies.foo1", "$cookie_foo1"] + - eq: ["$.cookies.foo3", "$cookie_foo3"] diff --git a/examples/postman_echo/debugtalk.py b/examples/postman_echo/debugtalk.py new file mode 100644 index 00000000..9ec3149b --- /dev/null +++ b/examples/postman_echo/debugtalk.py @@ -0,0 +1,5 @@ +from httprunner import __version__ + + +def get_httprunner_version(): + return __version__ From 5d8b09628ff150336106184e101c0b43713bfeea Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 19 Apr 2020 12:30:14 +0800 Subject: [PATCH 02/49] test: add testcase example, request methods testcase in hardcode --- .../postman_echo/request_methods/hardcode.yml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 examples/postman_echo/request_methods/hardcode.yml diff --git a/examples/postman_echo/request_methods/hardcode.yml b/examples/postman_echo/request_methods/hardcode.yml new file mode 100644 index 00000000..6cb3fde1 --- /dev/null +++ b/examples/postman_echo/request_methods/hardcode.yml @@ -0,0 +1,51 @@ +config: + name: "request methods testcase in hardcode" + base_url: "https://postman-echo.com" + verify: False + +teststeps: +- + name: get with params + request: + method: GET + url: /get + params: + foo1: bar1 + foo2: bar2 + headers: + User-Agent: HttpRunner/3.0 + validate: + - eq: ["status_code", 200] +- + name: post raw text + request: + method: POST + url: /post + headers: + User-Agent: HttpRunner/3.0 + Content-Type: "text/plain" + data: "This is expected to be sent back as part of response body." + validate: + - eq: ["status_code", 200] +- + name: post form data + request: + method: POST + url: /post + headers: + User-Agent: HttpRunner/3.0 + Content-Type: "application/x-www-form-urlencoded" + data: "foo1=bar1&foo2=bar2" + validate: + - eq: ["status_code", 200] +- + name: put request + request: + method: PUT + url: /put + headers: + User-Agent: HttpRunner/3.0 + Content-Type: "text/plain" + data: "This is expected to be sent back as part of response body." + validate: + - eq: ["status_code", 200] \ No newline at end of file From f1fa62b2c456641179ac90451236709114474cdf Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 19 Apr 2020 12:47:28 +0800 Subject: [PATCH 03/49] init v3: make http request without validation --- examples/__init__.py | 0 examples/postman_echo/__init__.py | 0 .../postman_echo/request_methods/__init__.py | 0 .../postman_echo/request_methods/hardcode.py | 79 ++++++++ httprunner/v3/__init__.py | 0 httprunner/v3/exceptions/__init__.py | 81 ++++++++ httprunner/v3/parser/__init__.py | 178 ++++++++++++++++++ httprunner/v3/runner/__init__.py | 38 ++++ httprunner/v3/schema/__init__.py | 60 ++++++ 9 files changed, 436 insertions(+) create mode 100644 examples/__init__.py create mode 100644 examples/postman_echo/__init__.py create mode 100644 examples/postman_echo/request_methods/__init__.py create mode 100644 examples/postman_echo/request_methods/hardcode.py create mode 100644 httprunner/v3/__init__.py create mode 100644 httprunner/v3/exceptions/__init__.py create mode 100644 httprunner/v3/parser/__init__.py create mode 100644 httprunner/v3/runner/__init__.py create mode 100644 httprunner/v3/schema/__init__.py diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/postman_echo/__init__.py b/examples/postman_echo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/postman_echo/request_methods/__init__.py b/examples/postman_echo/request_methods/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/postman_echo/request_methods/hardcode.py b/examples/postman_echo/request_methods/hardcode.py new file mode 100644 index 00000000..d1129ab6 --- /dev/null +++ b/examples/postman_echo/request_methods/hardcode.py @@ -0,0 +1,79 @@ +from httprunner.v3.runner import TestCaseRunner +from httprunner.v3.schema import TestsConfig, TestStep + + +class TestCaseRequestMethodsHardcode(TestCaseRunner): + config = TestsConfig(**{ + "name": "request methods testcase in hardcode", + "base_url": "https://postman-echo.com", + "verify": False + }) + + teststeps = [ + TestStep(**{ + "name": "get with params", + "request": { + "method": "GET", + "url": "/get", + "params": { + "foo1": "bar1", + "foo2": "bar2" + }, + "headers": { + "User-Agent": "HttpRunner/3.0" + } + }, + "validate": [ + {"eq": ["status_code", 200]} + ] + }), + TestStep(**{ + "name": "post raw text", + "request": { + "method": "POST", + "url": "/post", + "data": "This is expected to be sent back as part of response body.", + "headers": { + "User-Agent": "HttpRunner/3.0", + "Content-Type": "text/plain" + } + }, + "validate": [ + {"eq": ["status_code", 200]} + ] + }), + TestStep(**{ + "name": "post form data", + "request": { + "method": "POST", + "url": "/post", + "data": "foo1=bar1&foo2=bar2", + "headers": { + "User-Agent": "HttpRunner/3.0", + "Content-Type": "application/x-www-form-urlencoded" + } + }, + "validate": [ + {"eq": ["status_code", 200]} + ] + }), + TestStep(**{ + "name": "put request", + "request": { + "method": "PUT", + "url": "/put", + "data": "This is expected to be sent back as part of response body.", + "headers": { + "User-Agent": "HttpRunner/3.0", + "Content-Type": "text/plain" + } + }, + "validate": [ + {"eq": ["status_code", 200]} + ] + }) + ] + + +if __name__ == '__main__': + TestCaseRequestMethodsHardcode().run() diff --git a/httprunner/v3/__init__.py b/httprunner/v3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/httprunner/v3/exceptions/__init__.py b/httprunner/v3/exceptions/__init__.py new file mode 100644 index 00000000..77d1be52 --- /dev/null +++ b/httprunner/v3/exceptions/__init__.py @@ -0,0 +1,81 @@ +""" failure type exceptions + these exceptions will mark test as failure +""" + + +class MyBaseFailure(Exception): + pass + + +class ParseTestsFailure(MyBaseFailure): + pass + + +class ValidationFailure(MyBaseFailure): + pass + + +class ExtractFailure(MyBaseFailure): + pass + + +class SetupHooksFailure(MyBaseFailure): + pass + + +class TeardownHooksFailure(MyBaseFailure): + pass + + +""" error type exceptions + these exceptions will mark test as error +""" + + +class MyBaseError(Exception): + pass + + +class FileFormatError(MyBaseError): + pass + + +class ParamsError(MyBaseError): + pass + + +class NotFoundError(MyBaseError): + pass + + +class FileNotFound(FileNotFoundError, NotFoundError): + pass + + +class FunctionNotFound(NotFoundError): + pass + + +class VariableNotFound(NotFoundError): + pass + + +class EnvNotFound(NotFoundError): + pass + + +class CSVNotFound(NotFoundError): + pass + + +class ApiNotFound(NotFoundError): + pass + + +class TestcaseNotFound(NotFoundError): + pass + + +class SummaryEmpty(MyBaseError): + """ test result summary data is empty + """ diff --git a/httprunner/v3/parser/__init__.py b/httprunner/v3/parser/__init__.py new file mode 100644 index 00000000..1a7803a1 --- /dev/null +++ b/httprunner/v3/parser/__init__.py @@ -0,0 +1,178 @@ +import re +from typing import Any, Set +from typing import Dict + +from httprunner.v3.exceptions import ParamsError + +absolute_http_url_regexp = re.compile(r"^https?://", re.I) + +# use $$ to escape $ notation +dolloar_regex_compile = re.compile(r"\$\$") +# variable notation, e.g. ${var} or $var +variable_regex_compile = re.compile(r"\$\{(\w+)\}|\$(\w+)") +# function notation, e.g. ${func1($var_1, $var_3)} +function_regex_compile = re.compile(r"\$\{(\w+)\(([\$\w\.\-/\s=,]*)\)\}") + + +def build_url(base_url, path): + """ prepend url with base_url unless it's already an absolute URL """ + if absolute_http_url_regexp.match(path): + return path + elif base_url: + return "{}/{}".format(base_url.rstrip("/"), path.lstrip("/")) + else: + raise ParamsError("base url missed!") + + +def regex_findall_variables(content): + """ extract all variable names from content, which is in format $variable + + Args: + content (str): string content + + Returns: + list: variables list extracted from string content + + Examples: + >>> regex_findall_variables("$variable") + ["variable"] + + >>> regex_findall_variables("/blog/$postid") + ["postid"] + + >>> regex_findall_variables("/$var1/$var2") + ["var1", "var2"] + + >>> regex_findall_variables("abc") + [] + + """ + try: + vars_list = [] + for var_tuple in variable_regex_compile.findall(content): + vars_list.append( + var_tuple[0] or var_tuple[1] + ) + return vars_list + except TypeError: + return [] + + + +def extract_variables(content: Any) -> Set: + """ extract all variables in content recursively. + """ + if isinstance(content, (list, set, tuple)): + variables = set() + for item in content: + variables = variables | extract_variables(item) + return variables + + elif isinstance(content, dict): + variables = set() + for key, value in content.items(): + variables = variables | extract_variables(value) + return variables + + elif isinstance(content, str): + return set(regex_findall_variables(content)) + + return set() + + +def parse_string_variables(content, variables_mapping): + """ parse string content with variables mapping. + + Args: + content (str): string content to be parsed. + variables_mapping (dict): variables mapping. + + Returns: + str: parsed string content. + + Examples: + >>> content = "/api/users/$uid" + >>> variables_mapping = {"$uid": 1000} + >>> parse_string_variables(content, variables_mapping) + "/api/users/1000" + + """ + variables_list = extract_variables(content) + for variable_name in variables_list: + variable_value = variables_mapping[variable_name] + + # TODO: replace variable label from $var to {{var}} + if "${}".format(variable_name) == content: + # content is a variable + content = variable_value + else: + # content contains one or several variables + if not isinstance(variable_value, str): + variable_value = str(variable_value) + + content = content.replace( + "${}".format(variable_name), + variable_value, 1 + ) + + return content + + +def parse_content(content: Any, variables_mapping: Dict[str, Any] = None, functions_mapping=None): + """ parse content with evaluated variables mapping. + Notice: variables_mapping should not contain any variable or function. + """ + # TODO: refactor type check + if content is None or isinstance(content, (int, float, bool)): + return content + + elif isinstance(content, str): + # content is in string format here + variables_mapping = variables_mapping or {} + functions_mapping = functions_mapping or {} + content = content.strip() + + # replace functions with evaluated value + # Notice: _eval_content_functions must be called before _eval_content_variables + # content = parse_string_functions(content, variables_mapping, functions_mapping) + + # replace variables with binding value + content = parse_string_variables(content, variables_mapping) + + return content + + elif isinstance(content, (list, set, tuple)): + return [ + parse_content(item, variables_mapping) + for item in content + ] + + elif isinstance(content, dict): + parsed_content = {} + for key, value in content.items(): + parsed_key = parse_content(key, variables_mapping) + parsed_value = parse_content(value, variables_mapping) + parsed_content[parsed_key] = parsed_value + + return parsed_content + + return content + + +def parse_variables_mapping(variables_mapping: Dict[str, Any]): + + parsed_variables: Dict[str, Any] = {} + + while len(parsed_variables) != len(variables_mapping): + for var_name in variables_mapping: + + var_value = variables_mapping[var_name] + # variables = extract_variables(var_value) + + if var_name in parsed_variables: + continue + + parsed_value = parse_content(var_value, parsed_variables) + parsed_variables[var_name] = parsed_value + + return parsed_variables diff --git a/httprunner/v3/runner/__init__.py b/httprunner/v3/runner/__init__.py new file mode 100644 index 00000000..61638567 --- /dev/null +++ b/httprunner/v3/runner/__init__.py @@ -0,0 +1,38 @@ +from typing import List + +import requests + +from httprunner.v3.parser import build_url +from httprunner.v3.schema import TestsConfig, TestStep + + +class TestCaseRunner(object): + + config: TestsConfig = {} + teststeps: List[TestStep] = [] + session: requests.Session = None + + def with_session(self, s: requests.Session) -> "TestCaseRunner": + self.session = s + return self + + def with_variables(self, **variables) -> "TestCaseRunner": + self.config.variables.update(variables) + return self + + def run_step(self, step): + request_dict = step.request.dict() + + method = request_dict.pop("method") + url_path = request_dict.pop("url") + url = build_url(self.config.base_url, url_path) + + request_dict["json"] = request_dict.pop("req_json", {}) + + session = self.session or requests.Session() + resp = session.request(method, url, **request_dict) + + def run(self): + for step in self.teststeps: + step.variables.update(self.config.variables) + self.run_step(step) diff --git a/httprunner/v3/schema/__init__.py b/httprunner/v3/schema/__init__.py new file mode 100644 index 00000000..fcaac363 --- /dev/null +++ b/httprunner/v3/schema/__init__.py @@ -0,0 +1,60 @@ +from enum import Enum +from typing import Any +from typing import Dict, List, Text, Union + +from pydantic import BaseModel, Field +from pydantic import HttpUrl + +Name = Text +Url = Text +BaseUrl = Union[HttpUrl, Text] +Variables = Dict[Text, Any] +Headers = Dict[Text, Text] +Verify = bool +Hook = List[Text] +Export = List[Text] +Validate = List[Dict] +Env = Dict[Text, Any] + + +class MethodEnum(Text, Enum): + GET = 'GET' + POST = 'POST' + PUT = "PUT" + DELETE = "DELETE" + HEAD = "HEAD" + OPTIONS = "OPTIONS" + PATCH = "PATCH" + CONNECT = "CONNECT" + TRACE = "TRACE" + + +class TestsConfig(BaseModel): + name: Name + verify: Verify = False + base_url: BaseUrl = "" + variables: Variables = {} + setup_hooks: Hook = [] + teardown_hooks: Hook = [] + export: Export = [] + + +class Request(BaseModel): + method: MethodEnum = MethodEnum.GET + url: Url + params: Dict[Text, Text] = {} + headers: Headers = {} + req_json: Dict = Field({}, alias="json") + data: Union[Text, Dict[Text, Any]] = "" + cookies: Dict[Text, Text] = {} + timeout: int = 120 + allow_redirects: bool = True + verify: Verify = False + + +class TestStep(BaseModel): + name: Name + request: Request + variables: Variables = {} + extract: Union[Dict[Text, Text], List[Text]] = {} + validation: Validate = Field([], alias="validate") From 9aef959d7c55d07b4f9cd31d70858d5a3f667064 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 19 Apr 2020 17:37:23 +0800 Subject: [PATCH 04/49] change v3: add basic validation --- .../{hardcode.py => hardcode_test.py} | 3 +- .../v3/{parser/__init__.py => parser.py} | 7 +- httprunner/v3/response.py | 13 ++ .../v3/{runner/__init__.py => runner.py} | 25 ++- httprunner/v3/validator.py | 145 ++++++++++++++++++ poetry.lock | 14 +- pyproject.toml | 1 + 7 files changed, 200 insertions(+), 8 deletions(-) rename examples/postman_echo/request_methods/{hardcode.py => hardcode_test.py} (95%) rename httprunner/v3/{parser/__init__.py => parser.py} (97%) create mode 100644 httprunner/v3/response.py rename httprunner/v3/{runner/__init__.py => runner.py} (62%) create mode 100644 httprunner/v3/validator.py diff --git a/examples/postman_echo/request_methods/hardcode.py b/examples/postman_echo/request_methods/hardcode_test.py similarity index 95% rename from examples/postman_echo/request_methods/hardcode.py rename to examples/postman_echo/request_methods/hardcode_test.py index d1129ab6..52f7899b 100644 --- a/examples/postman_echo/request_methods/hardcode.py +++ b/examples/postman_echo/request_methods/hardcode_test.py @@ -24,7 +24,8 @@ class TestCaseRequestMethodsHardcode(TestCaseRunner): } }, "validate": [ - {"eq": ["status_code", 200]} + {"eq": ["status_code", 200]}, + {"eq": ["headers.Server", "nginx"]} ] }), TestStep(**{ diff --git a/httprunner/v3/parser/__init__.py b/httprunner/v3/parser.py similarity index 97% rename from httprunner/v3/parser/__init__.py rename to httprunner/v3/parser.py index 1a7803a1..c1ed97a9 100644 --- a/httprunner/v3/parser/__init__.py +++ b/httprunner/v3/parser.py @@ -1,8 +1,8 @@ import re -from typing import Any, Set +from typing import Any, Set, Text from typing import Dict -from httprunner.v3.exceptions import ParamsError +from httprunner.v3 import exceptions absolute_http_url_regexp = re.compile(r"^https?://", re.I) @@ -21,7 +21,7 @@ def build_url(base_url, path): elif base_url: return "{}/{}".format(base_url.rstrip("/"), path.lstrip("/")) else: - raise ParamsError("base url missed!") + raise exceptions.ParamsError("base url missed!") def regex_findall_variables(content): @@ -58,7 +58,6 @@ def regex_findall_variables(content): return [] - def extract_variables(content: Any) -> Set: """ extract all variables in content recursively. """ diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py new file mode 100644 index 00000000..422218bf --- /dev/null +++ b/httprunner/v3/response.py @@ -0,0 +1,13 @@ +import requests + + +class ResponseObject(object): + + def __init__(self, resp_obj: requests.Response): + """ initialize with a requests.Response object + + Args: + resp_obj (instance): requests.Response instance + + """ + self.obj = resp_obj diff --git a/httprunner/v3/runner/__init__.py b/httprunner/v3/runner.py similarity index 62% rename from httprunner/v3/runner/__init__.py rename to httprunner/v3/runner.py index 61638567..7d4a12f2 100644 --- a/httprunner/v3/runner/__init__.py +++ b/httprunner/v3/runner.py @@ -2,8 +2,12 @@ from typing import List import requests +from loguru import logger + from httprunner.v3.parser import build_url from httprunner.v3.schema import TestsConfig, TestStep +from httprunner.v3.validator import Validator +from httprunner.v3.response import ResponseObject class TestCaseRunner(object): @@ -21,18 +25,35 @@ class TestCaseRunner(object): return self def run_step(self, step): - request_dict = step.request.dict() + logger.info(f"run step: {step.name}") + # prepare arguments + request_dict = step.request.dict() method = request_dict.pop("method") url_path = request_dict.pop("url") url = build_url(self.config.base_url, url_path) request_dict["json"] = request_dict.pop("req_json", {}) + logger.info(f"{method} {url}") + logger.debug(f"request kwargs(raw): {request_dict}") + + # request session = self.session or requests.Session() resp = session.request(method, url, **request_dict) - def run(self): + # validate + resp_obj = ResponseObject(resp) + validator = Validator(resp_obj) + validators = step.validation + validator.validate(validators) + + def test_start(self): + """main entrance""" for step in self.teststeps: step.variables.update(self.config.variables) self.run_step(step) + + def run(self): + """main entrance alias for test_start""" + return self.test_start() diff --git a/httprunner/v3/validator.py b/httprunner/v3/validator.py new file mode 100644 index 00000000..04940b13 --- /dev/null +++ b/httprunner/v3/validator.py @@ -0,0 +1,145 @@ +from typing import Text + +import jmespath +from loguru import logger + +from httprunner.v3.exceptions import ParamsError, ValidationFailure +from httprunner.v3.response import ResponseObject + + +def get_uniform_comparator(comparator: Text): + """ convert comparator alias to uniform name + """ + if comparator in ["eq", "equals", "==", "is"]: + return "equals" + elif comparator in ["lt", "less_than"]: + return "less_than" + elif comparator in ["le", "less_than_or_equals"]: + return "less_than_or_equals" + elif comparator in ["gt", "greater_than"]: + return "greater_than" + elif comparator in ["ge", "greater_than_or_equals"]: + return "greater_than_or_equals" + elif comparator in ["ne", "not_equals"]: + return "not_equals" + elif comparator in ["str_eq", "string_equals"]: + return "string_equals" + elif comparator in ["len_eq", "length_equals", "count_eq"]: + return "length_equals" + elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]: + return "length_greater_than" + elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", + "count_greater_than_or_equals"]: + return "length_greater_than_or_equals" + elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]: + return "length_less_than" + elif comparator in ["len_le", "count_le", "length_less_than_or_equals", + "count_less_than_or_equals"]: + return "length_less_than_or_equals" + else: + return comparator + + +def uniform_validator(validator): + """ unify validator + + Args: + validator (dict): validator maybe in two formats: + + format1: this is kept for compatiblity with the previous versions. + {"check": "status_code", "assert": "eq", "expect": 201} + {"check": "$resp_body_success", "assert": "eq", "expect": True} + format2: recommended new version, {assert: [check_item, expected_value]} + {'eq': ['status_code', 201]} + {'eq': ['$resp_body_success', True]} + + Returns + dict: validator info + + { + "check": "status_code", + "expect": 201, + "assert": "equals" + } + + """ + if not isinstance(validator, dict): + raise ParamsError(f"invalid validator: {validator}") + + if "check" in validator and "expect" in validator: + # format1 + check_item = validator["check"] + expect_value = validator["expect"] + comparator = validator.get("comparator", "eq") + + elif len(validator) == 1: + # format2 + comparator = list(validator.keys())[0] + compare_values = validator[comparator] + + if not isinstance(compare_values, list) or len(compare_values) != 2: + raise ParamsError(f"invalid validator: {validator}") + + check_item, expect_value = compare_values + + else: + raise ParamsError(f"invalid validator: {validator}") + + # uniform comparator, e.g. lt => less_than, eq => equals + assert_method = get_uniform_comparator(comparator) + + return { + "check": check_item, + "expect": expect_value, + "assert": assert_method + } + + +class AssertMethods(object): + + @staticmethod + def equals(actual_value, expect_value): + assert actual_value == expect_value + + @staticmethod + def less_than(actual_value, expect_value): + assert actual_value < expect_value + + @staticmethod + def greater_than(actual_value, expect_value): + assert actual_value > expect_value + + +class Validator(object): + + def __init__(self, resp_obj: ResponseObject): + self.resp_meta = { + "status_code": resp_obj.obj.status_code, + "headers": resp_obj.obj.headers, + "body": resp_obj.obj.json() + } + + def validate(self, validators): + + for v in validators: + u_validator = uniform_validator(v) + field = u_validator["check"] + assert_method = u_validator["assert"] + expect_value = u_validator["expect"] + actual_value = jmespath.search(field, self.resp_meta) + + msg = f"assert {field} {assert_method} {expect_value}" + + try: + assert_func = getattr(AssertMethods, assert_method) + except AttributeError: + raise ParamsError(f"Assert Method not supported: {assert_method}") + + try: + assert_func(actual_value, expect_value) + msg += " - success" + logger.info(msg) + except AssertionError: + msg += " - fail" + logger.error(msg) + raise ValidationFailure(f"assert {field}: {actual_value} {assert_method} {expect_value}") diff --git a/poetry.lock b/poetry.lock index 3dfab901..9d54bc93 100644 --- a/poetry.lock +++ b/poetry.lock @@ -173,6 +173,14 @@ MarkupSafe = ">=0.23" [package.extras] i18n = ["Babel (>=0.8)"] +[[package]] +category = "main" +description = "JSON Matching Expressions" +name = "jmespath" +optional = false +python-versions = "*" +version = "0.9.5" + [[package]] category = "main" description = "An XPath for JSON" @@ -345,7 +353,7 @@ version = "1.0.1" dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"] [metadata] -content-hash = "e1204ede1ab227bc33783b362d866c2a0b1fb8faba283216b2973e2261b0b966" +content-hash = "85e388b2b80681f40a72f5ec0a1746a5a7e834c63fbfc9e2a07058bebb6f6c29" python-versions = "^3.6" [metadata.files] @@ -470,6 +478,10 @@ jinja2 = [ {file = "Jinja2-2.10.3-py2.py3-none-any.whl", hash = "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f"}, {file = "Jinja2-2.10.3.tar.gz", hash = "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de"}, ] +jmespath = [ + {file = "jmespath-0.9.5-py2.py3-none-any.whl", hash = "sha256:695cb76fa78a10663425d5b73ddc5714eb711157e52704d69be03b1a02ba4fec"}, + {file = "jmespath-0.9.5.tar.gz", hash = "sha256:cca55c8d153173e21baa59983015ad0daf603f9cb799904ff057bfb8ff8dc2d9"}, +] jsonpath = [ {file = "jsonpath-0.82.tar.gz", hash = "sha256:46d3fd2016cd5b842283d547877a02c418a0fe9aa7a6b0ae344115a2c990fef4"}, ] diff --git a/pyproject.toml b/pyproject.toml index 3de0634c..9c47e387 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ filetype = "^1.0.5" jsonpath = "^0.82" pydantic = "^1.4" loguru = "^0.4.1" +jmespath = "^0.9.5" [tool.poetry.dev-dependencies] flask = "<1.0.0" From 1ce09d223cbd2c04b28ad7c0989b639dd344232a Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 19 Apr 2020 18:19:01 +0800 Subject: [PATCH 05/49] change v3: add basic extract --- .../request_methods/hardcode_test.py | 3 ++ httprunner/v3/response.py | 50 ++++++++++++++++++- httprunner/v3/runner.py | 19 ++++--- httprunner/v3/schema/__init__.py | 2 +- httprunner/v3/validator.py | 41 +-------------- 5 files changed, 66 insertions(+), 49 deletions(-) diff --git a/examples/postman_echo/request_methods/hardcode_test.py b/examples/postman_echo/request_methods/hardcode_test.py index 52f7899b..0c269b0b 100644 --- a/examples/postman_echo/request_methods/hardcode_test.py +++ b/examples/postman_echo/request_methods/hardcode_test.py @@ -23,6 +23,9 @@ class TestCaseRequestMethodsHardcode(TestCaseRunner): "User-Agent": "HttpRunner/3.0" } }, + "extract": { + "server": "headers.Server" + }, "validate": [ {"eq": ["status_code", 200]}, {"eq": ["headers.Server", "nginx"]} diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py index 422218bf..5dbf9577 100644 --- a/httprunner/v3/response.py +++ b/httprunner/v3/response.py @@ -1,4 +1,11 @@ +from typing import Dict, Text, Any + +import jmespath import requests +from loguru import logger + +from httprunner.v3.exceptions import ParamsError, ValidationFailure +from httprunner.v3.validator import uniform_validator, AssertMethods class ResponseObject(object): @@ -10,4 +17,45 @@ class ResponseObject(object): resp_obj (instance): requests.Response instance """ - self.obj = resp_obj + self.resp_obj_meta = { + "status_code": resp_obj.status_code, + "headers": resp_obj.headers, + "body": resp_obj.json() + } + + def validate(self, validators): + + for v in validators: + u_validator = uniform_validator(v) + field = u_validator["check"] + assert_method = u_validator["assert"] + expect_value = u_validator["expect"] + actual_value = jmespath.search(field, self.resp_obj_meta) + + msg = f"assert {field} {assert_method} {expect_value}" + + try: + assert_func = getattr(AssertMethods, assert_method) + except AttributeError: + raise ParamsError(f"Assert Method not supported: {assert_method}") + + try: + assert_func(actual_value, expect_value) + msg += " - success" + logger.info(msg) + except AssertionError: + msg += " - fail" + logger.error(msg) + raise ValidationFailure(f"assert {field}: {actual_value} {assert_method} {expect_value}") + + def extract(self, extractors: Dict[Text, Text]) -> Dict[Text, Any]: + if not extractors: + return {} + + extract_mapping = {} + for key, field in extractors.items(): + field_value = jmespath.search(field, self.resp_obj_meta) + extract_mapping[key] = field_value + + logger.info(f"extract mapping: {extract_mapping}") + return extract_mapping diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 7d4a12f2..49fb6303 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -1,13 +1,11 @@ from typing import List import requests - from loguru import logger from httprunner.v3.parser import build_url -from httprunner.v3.schema import TestsConfig, TestStep -from httprunner.v3.validator import Validator from httprunner.v3.response import ResponseObject +from httprunner.v3.schema import TestsConfig, TestStep class TestCaseRunner(object): @@ -41,18 +39,25 @@ class TestCaseRunner(object): # request session = self.session or requests.Session() resp = session.request(method, url, **request_dict) + resp_obj = ResponseObject(resp) # validate - resp_obj = ResponseObject(resp) - validator = Validator(resp_obj) validators = step.validation - validator.validate(validators) + resp_obj.validate(validators) + + # extract + extractors = step.extract + extract_mapping = resp_obj.extract(extractors) + return extract_mapping def test_start(self): """main entrance""" + session_variables = {} for step in self.teststeps: step.variables.update(self.config.variables) - self.run_step(step) + step.variables.update(session_variables) + extract_mapping = self.run_step(step) + session_variables.update(extract_mapping) def run(self): """main entrance alias for test_start""" diff --git a/httprunner/v3/schema/__init__.py b/httprunner/v3/schema/__init__.py index fcaac363..778a217a 100644 --- a/httprunner/v3/schema/__init__.py +++ b/httprunner/v3/schema/__init__.py @@ -56,5 +56,5 @@ class TestStep(BaseModel): name: Name request: Request variables: Variables = {} - extract: Union[Dict[Text, Text], List[Text]] = {} + extract: Dict[Text, Text] = {} validation: Validate = Field([], alias="validate") diff --git a/httprunner/v3/validator.py b/httprunner/v3/validator.py index 04940b13..cc0b39b9 100644 --- a/httprunner/v3/validator.py +++ b/httprunner/v3/validator.py @@ -1,10 +1,6 @@ from typing import Text -import jmespath -from loguru import logger - -from httprunner.v3.exceptions import ParamsError, ValidationFailure -from httprunner.v3.response import ResponseObject +from httprunner.v3.exceptions import ParamsError def get_uniform_comparator(comparator: Text): @@ -108,38 +104,3 @@ class AssertMethods(object): @staticmethod def greater_than(actual_value, expect_value): assert actual_value > expect_value - - -class Validator(object): - - def __init__(self, resp_obj: ResponseObject): - self.resp_meta = { - "status_code": resp_obj.obj.status_code, - "headers": resp_obj.obj.headers, - "body": resp_obj.obj.json() - } - - def validate(self, validators): - - for v in validators: - u_validator = uniform_validator(v) - field = u_validator["check"] - assert_method = u_validator["assert"] - expect_value = u_validator["expect"] - actual_value = jmespath.search(field, self.resp_meta) - - msg = f"assert {field} {assert_method} {expect_value}" - - try: - assert_func = getattr(AssertMethods, assert_method) - except AttributeError: - raise ParamsError(f"Assert Method not supported: {assert_method}") - - try: - assert_func(actual_value, expect_value) - msg += " - success" - logger.info(msg) - except AssertionError: - msg += " - fail" - logger.error(msg) - raise ValidationFailure(f"assert {field}: {actual_value} {assert_method} {expect_value}") From 3051a60731834bed093914fa9c7a47dbf2a2d195 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 20 Apr 2020 12:03:08 +0800 Subject: [PATCH 06/49] v3 feat: support teststep variables --- .../request_methods/with_functions.yml | 0 .../request_methods/with_variables.yml | 53 ++++++++++++ .../request_methods/with_variables_test.py | 80 +++++++++++++++++++ httprunner/v3/parser.py | 4 +- httprunner/v3/runner.py | 17 ++-- 5 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 examples/postman_echo/request_methods/with_functions.yml create mode 100644 examples/postman_echo/request_methods/with_variables.yml create mode 100644 examples/postman_echo/request_methods/with_variables_test.py diff --git a/examples/postman_echo/request_methods/with_functions.yml b/examples/postman_echo/request_methods/with_functions.yml new file mode 100644 index 00000000..e69de29b diff --git a/examples/postman_echo/request_methods/with_variables.yml b/examples/postman_echo/request_methods/with_variables.yml new file mode 100644 index 00000000..050d7910 --- /dev/null +++ b/examples/postman_echo/request_methods/with_variables.yml @@ -0,0 +1,53 @@ +config: + name: "request methods testcase with variables" + base_url: "https://postman-echo.com" + verify: False + +teststeps: +- + name: get with params + variables: + foo1: bar1 + foo2: bar2 + request: + method: GET + url: /get + params: + foo1: $foo1 + foo2: $foo2 + headers: + User-Agent: HttpRunner/3.0 + validate: + - eq: ["status_code", 200] + - eq: ["body.args.foo1", "bar1"] + - eq: ["body.args.foo2", "bar2"] +- + name: post raw text + variables: + foo1: "hello world" + request: + method: POST + url: /post + headers: + User-Agent: HttpRunner/3.0 + Content-Type: "text/plain" + data: "This is expected to be sent back as part of response body: $foo1." + validate: + - eq: ["status_code", 200] + - eq: ["body.data", "This is expected to be sent back as part of response body: hello world."] +- + name: post form data + variables: + foo1: bar1 + foo2: bar2 + request: + method: POST + url: /post + headers: + User-Agent: HttpRunner/3.0 + Content-Type: "application/x-www-form-urlencoded" + data: "foo1=$foo1&foo2=$foo2" + validate: + - eq: ["status_code", 200] + - eq: ["body.form.foo1", "bar1"] + - eq: ["body.form.foo2", "bar2"] diff --git a/examples/postman_echo/request_methods/with_variables_test.py b/examples/postman_echo/request_methods/with_variables_test.py new file mode 100644 index 00000000..cdec850e --- /dev/null +++ b/examples/postman_echo/request_methods/with_variables_test.py @@ -0,0 +1,80 @@ +from httprunner.v3.runner import TestCaseRunner +from httprunner.v3.schema import TestsConfig, TestStep + + +class TestCaseRequestMethodsWithVariables(TestCaseRunner): + config = TestsConfig(**{ + "name": "request methods testcase with variables", + "base_url": "https://postman-echo.com", + "verify": False + }) + + teststeps = [ + TestStep(**{ + "name": "get with params", + "variables": { + "foo1": "bar1", + "foo2": "bar2" + }, + "request": { + "method": "GET", + "url": "/get", + "params": { + "foo1": "$foo1", + "foo2": "$foo2" + }, + "headers": { + "User-Agent": "HttpRunner/3.0" + } + }, + "validate": [ + {"eq": ["status_code", 200]}, + {"eq": ["body.args.foo1", "bar1"]}, + {"eq": ["body.args.foo2", "bar2"]} + ] + }), + TestStep(**{ + "name": "post raw text", + "variables": { + "foo1": "hello world" + }, + "request": { + "method": "POST", + "url": "/post", + "data": "This is expected to be sent back as part of response body: $foo1.", + "headers": { + "User-Agent": "HttpRunner/3.0", + "Content-Type": "text/plain" + } + }, + "validate": [ + {"eq": ["status_code", 200]}, + {"eq": ["body.data", "This is expected to be sent back as part of response body: hello world."]}, + ] + }), + TestStep(**{ + "name": "post form data", + "variables": { + "foo1": "bar1", + "foo2": "bar2" + }, + "request": { + "method": "POST", + "url": "/post", + "data": "foo1=$foo1&foo2=$foo2", + "headers": { + "User-Agent": "HttpRunner/3.0", + "Content-Type": "application/x-www-form-urlencoded" + } + }, + "validate": [ + {"eq": ["status_code", 200]}, + {"eq": ["body.form.foo1", "bar1"]}, + {"eq": ["body.form.foo2", "bar2"]} + ] + }) + ] + + +if __name__ == '__main__': + TestCaseRequestMethodsWithVariables().run() diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index c1ed97a9..c3365490 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -101,7 +101,7 @@ def parse_string_variables(content, variables_mapping): variable_value = variables_mapping[variable_name] # TODO: replace variable label from $var to {{var}} - if "${}".format(variable_name) == content: + if f"${variable_name}" == content: # content is a variable content = variable_value else: @@ -110,7 +110,7 @@ def parse_string_variables(content, variables_mapping): variable_value = str(variable_value) content = content.replace( - "${}".format(variable_name), + f"${variable_name}", variable_value, 1 ) diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 49fb6303..fc237f71 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -3,7 +3,7 @@ from typing import List import requests from loguru import logger -from httprunner.v3.parser import build_url +from httprunner.v3.parser import build_url, parse_content from httprunner.v3.response import ResponseObject from httprunner.v3.schema import TestsConfig, TestStep @@ -25,20 +25,23 @@ class TestCaseRunner(object): def run_step(self, step): logger.info(f"run step: {step.name}") - # prepare arguments + # parse request_dict = step.request.dict() - method = request_dict.pop("method") - url_path = request_dict.pop("url") + parsed_request_dict = parse_content(request_dict, step.variables) + + # prepare arguments + method = parsed_request_dict.pop("method") + url_path = parsed_request_dict.pop("url") url = build_url(self.config.base_url, url_path) - request_dict["json"] = request_dict.pop("req_json", {}) + parsed_request_dict["json"] = parsed_request_dict.pop("req_json", {}) logger.info(f"{method} {url}") - logger.debug(f"request kwargs(raw): {request_dict}") + logger.debug(f"request kwargs(raw): {parsed_request_dict}") # request session = self.session or requests.Session() - resp = session.request(method, url, **request_dict) + resp = session.request(method, url, **parsed_request_dict) resp_obj = ResponseObject(resp) # validate From 39b8f74b2745d4cdbd47a2bc01494d1f43015dd2 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 20 Apr 2020 12:49:53 +0800 Subject: [PATCH 07/49] v3 test example: support session variables --- .../postman_echo/request_methods/with_functions.yml | 0 .../postman_echo/request_methods/with_variables.yml | 8 +++++--- .../request_methods/with_variables_test.py | 11 +++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) delete mode 100644 examples/postman_echo/request_methods/with_functions.yml diff --git a/examples/postman_echo/request_methods/with_functions.yml b/examples/postman_echo/request_methods/with_functions.yml deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/postman_echo/request_methods/with_variables.yml b/examples/postman_echo/request_methods/with_variables.yml index 050d7910..008dd007 100644 --- a/examples/postman_echo/request_methods/with_variables.yml +++ b/examples/postman_echo/request_methods/with_variables.yml @@ -1,5 +1,7 @@ config: name: "request methods testcase with variables" + variables: + foo1: session_bar1 base_url: "https://postman-echo.com" verify: False @@ -19,7 +21,7 @@ teststeps: User-Agent: HttpRunner/3.0 validate: - eq: ["status_code", 200] - - eq: ["body.args.foo1", "bar1"] + - eq: ["body.args.foo1", "session_bar1"] - eq: ["body.args.foo2", "bar2"] - name: post raw text @@ -34,7 +36,7 @@ teststeps: data: "This is expected to be sent back as part of response body: $foo1." validate: - eq: ["status_code", 200] - - eq: ["body.data", "This is expected to be sent back as part of response body: hello world."] + - eq: ["body.data", "This is expected to be sent back as part of response body: session_bar1."] - name: post form data variables: @@ -49,5 +51,5 @@ teststeps: data: "foo1=$foo1&foo2=$foo2" validate: - eq: ["status_code", 200] - - eq: ["body.form.foo1", "bar1"] + - eq: ["body.form.foo1", "session_bar1"] - eq: ["body.form.foo2", "bar2"] diff --git a/examples/postman_echo/request_methods/with_variables_test.py b/examples/postman_echo/request_methods/with_variables_test.py index cdec850e..c68c99fa 100644 --- a/examples/postman_echo/request_methods/with_variables_test.py +++ b/examples/postman_echo/request_methods/with_variables_test.py @@ -5,6 +5,9 @@ from httprunner.v3.schema import TestsConfig, TestStep class TestCaseRequestMethodsWithVariables(TestCaseRunner): config = TestsConfig(**{ "name": "request methods testcase with variables", + "variables": { + "foo1": "session_bar1" + }, "base_url": "https://postman-echo.com", "verify": False }) @@ -29,7 +32,7 @@ class TestCaseRequestMethodsWithVariables(TestCaseRunner): }, "validate": [ {"eq": ["status_code", 200]}, - {"eq": ["body.args.foo1", "bar1"]}, + {"eq": ["body.args.foo1", "session_bar1"]}, {"eq": ["body.args.foo2", "bar2"]} ] }), @@ -49,13 +52,13 @@ class TestCaseRequestMethodsWithVariables(TestCaseRunner): }, "validate": [ {"eq": ["status_code", 200]}, - {"eq": ["body.data", "This is expected to be sent back as part of response body: hello world."]}, + {"eq": ["body.data", "This is expected to be sent back as part of response body: session_bar1."]}, ] }), TestStep(**{ "name": "post form data", "variables": { - "foo1": "bar1", + "foo1": "session_bar1", "foo2": "bar2" }, "request": { @@ -69,7 +72,7 @@ class TestCaseRequestMethodsWithVariables(TestCaseRunner): }, "validate": [ {"eq": ["status_code", 200]}, - {"eq": ["body.form.foo1", "bar1"]}, + {"eq": ["body.form.foo1", "session_bar1"]}, {"eq": ["body.form.foo2", "bar2"]} ] }) From bd7b84d42cb940d315896476cc836e5aed70a142 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 20 Apr 2020 13:26:40 +0800 Subject: [PATCH 08/49] v3 feat: support extract session variables --- .../request_methods/with_variables.yml | 11 ++++++---- .../request_methods/with_variables_test.py | 17 +++++++++----- httprunner/v3/parser.py | 22 +++++++++++++------ httprunner/v3/runner.py | 8 ++++++- 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/examples/postman_echo/request_methods/with_variables.yml b/examples/postman_echo/request_methods/with_variables.yml index 008dd007..625e240f 100644 --- a/examples/postman_echo/request_methods/with_variables.yml +++ b/examples/postman_echo/request_methods/with_variables.yml @@ -10,7 +10,7 @@ teststeps: name: get with params variables: foo1: bar1 - foo2: bar2 + foo2: session_bar2 request: method: GET url: /get @@ -19,24 +19,27 @@ teststeps: foo2: $foo2 headers: User-Agent: HttpRunner/3.0 + extract: + session_foo2: "body.args.foo2" validate: - eq: ["status_code", 200] - eq: ["body.args.foo1", "session_bar1"] - - eq: ["body.args.foo2", "bar2"] + - eq: ["body.args.foo2", "session_bar2"] - name: post raw text variables: foo1: "hello world" + foo3: "$session_foo2" request: method: POST url: /post headers: User-Agent: HttpRunner/3.0 Content-Type: "text/plain" - data: "This is expected to be sent back as part of response body: $foo1." + data: "This is expected to be sent back as part of response body: $foo1-$foo3." validate: - eq: ["status_code", 200] - - eq: ["body.data", "This is expected to be sent back as part of response body: session_bar1."] + - eq: ["body.data", "This is expected to be sent back as part of response body: session_bar1-session_bar2."] - name: post form data variables: diff --git a/examples/postman_echo/request_methods/with_variables_test.py b/examples/postman_echo/request_methods/with_variables_test.py index c68c99fa..df29953f 100644 --- a/examples/postman_echo/request_methods/with_variables_test.py +++ b/examples/postman_echo/request_methods/with_variables_test.py @@ -17,7 +17,7 @@ class TestCaseRequestMethodsWithVariables(TestCaseRunner): "name": "get with params", "variables": { "foo1": "bar1", - "foo2": "bar2" + "foo2": "session_bar2" }, "request": { "method": "GET", @@ -30,21 +30,25 @@ class TestCaseRequestMethodsWithVariables(TestCaseRunner): "User-Agent": "HttpRunner/3.0" } }, + "extract": { + "session_foo2": "body.args.foo2" + }, "validate": [ {"eq": ["status_code", 200]}, {"eq": ["body.args.foo1", "session_bar1"]}, - {"eq": ["body.args.foo2", "bar2"]} + {"eq": ["body.args.foo2", "session_bar2"]} ] }), TestStep(**{ "name": "post raw text", "variables": { - "foo1": "hello world" + "foo1": "hello world", + "foo3": "$session_foo2" }, "request": { "method": "POST", "url": "/post", - "data": "This is expected to be sent back as part of response body: $foo1.", + "data": "This is expected to be sent back as part of response body: $foo1-$foo3.", "headers": { "User-Agent": "HttpRunner/3.0", "Content-Type": "text/plain" @@ -52,7 +56,10 @@ class TestCaseRequestMethodsWithVariables(TestCaseRunner): }, "validate": [ {"eq": ["status_code", 200]}, - {"eq": ["body.data", "This is expected to be sent back as part of response body: session_bar1."]}, + {"eq": [ + "body.data", + "This is expected to be sent back as part of response body: session_bar1-session_bar2." + ]}, ] }), TestStep(**{ diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index c3365490..9e1b2ad3 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -3,6 +3,7 @@ from typing import Any, Set, Text from typing import Dict from httprunner.v3 import exceptions +from httprunner.v3.exceptions import VariableNotFound absolute_http_url_regexp = re.compile(r"^https?://", re.I) @@ -98,7 +99,10 @@ def parse_string_variables(content, variables_mapping): """ variables_list = extract_variables(content) for variable_name in variables_list: - variable_value = variables_mapping[variable_name] + try: + variable_value = variables_mapping[variable_name] + except KeyError: + raise VariableNotFound(f"{variable_name} not in {variables_mapping}") # TODO: replace variable label from $var to {{var}} if f"${variable_name}" == content: @@ -158,20 +162,24 @@ def parse_content(content: Any, variables_mapping: Dict[str, Any] = None, functi return content -def parse_variables_mapping(variables_mapping: Dict[str, Any]): +def parse_variables_mapping(variables_mapping: Dict[Text, Any]) -> Dict[Text, Any]: - parsed_variables: Dict[str, Any] = {} + parsed_variables: Dict[Text, Any] = {} while len(parsed_variables) != len(variables_mapping): for var_name in variables_mapping: - var_value = variables_mapping[var_name] - # variables = extract_variables(var_value) - if var_name in parsed_variables: continue - parsed_value = parse_content(var_value, parsed_variables) + var_value = variables_mapping[var_name] + # variables = extract_variables(var_value) + + try: + parsed_value = parse_content(var_value, parsed_variables) + except VariableNotFound: + continue + parsed_variables[var_name] = parsed_value return parsed_variables diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index fc237f71..12d28059 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -3,7 +3,7 @@ from typing import List import requests from loguru import logger -from httprunner.v3.parser import build_url, parse_content +from httprunner.v3.parser import build_url, parse_content, parse_variables_mapping from httprunner.v3.response import ResponseObject from httprunner.v3.schema import TestsConfig, TestStep @@ -57,9 +57,15 @@ class TestCaseRunner(object): """main entrance""" session_variables = {} for step in self.teststeps: + # update with config variables step.variables.update(self.config.variables) + # update with session variables extracted from former step step.variables.update(session_variables) + # parse variables + step.variables = parse_variables_mapping(step.variables) + # run step extract_mapping = self.run_step(step) + # save extracted variables to session variables session_variables.update(extract_mapping) def run(self): From 4d69e0c8588e71f1739e2dd1183ab3438d4b8dae Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 20 Apr 2020 15:44:26 +0800 Subject: [PATCH 09/49] fix parser: check if reference variable itself check if reference variable not in variables_mapping --- httprunner/v3/parser.py | 20 +++++++++++++++++++- httprunner/v3/parser_test.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 httprunner/v3/parser_test.py diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index 9e1b2ad3..f6ff3a58 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -173,7 +173,25 @@ def parse_variables_mapping(variables_mapping: Dict[Text, Any]) -> Dict[Text, An continue var_value = variables_mapping[var_name] - # variables = extract_variables(var_value) + variables = extract_variables(var_value) + + # check if reference variable itself + if var_name in variables: + # e.g. + # variables_mapping = {"token": "abc$token"} + # variables_mapping = {"key": ["$key", 2]} + raise exceptions.VariableNotFound(var_name) + + # check if reference variable not in variables_mapping + not_defined_variables = [ + v_name + for v_name in variables + if v_name not in variables_mapping + ] + if not_defined_variables: + # e.g. {"varA": "123$varB", "varB": "456$varC"} + # e.g. {"varC": "${sum_two($a, $b)}"} + raise VariableNotFound(not_defined_variables) try: parsed_value = parse_content(var_value, parsed_variables) diff --git a/httprunner/v3/parser_test.py b/httprunner/v3/parser_test.py new file mode 100644 index 00000000..f4a02068 --- /dev/null +++ b/httprunner/v3/parser_test.py @@ -0,0 +1,30 @@ +import unittest + +from httprunner.v3.parser import parse_variables_mapping +from httprunner.v3.exceptions import VariableNotFound + + +class TestParserBasic(unittest.TestCase): + + def test_parse_variables_mapping(self): + variables = { + "varA": "$varB", + "varB": "$varC", + "varC": "123", + "a": 1, + "b": 2 + } + parsed_variables = parse_variables_mapping(variables) + print(parsed_variables) + self.assertEqual(parsed_variables["varA"], "123") + self.assertEqual(parsed_variables["varB"], "123") + + def test_parse_variables_mapping_exception(self): + variables = { + "varA": "$varB", + "varB": "$varC", + "a": 1, + "b": 2 + } + with self.assertRaises(VariableNotFound): + parse_variables_mapping(variables) From 36f9af441cccdd1a4499cb83c03167a622ede320 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 20 Apr 2020 17:06:21 +0800 Subject: [PATCH 10/49] v3 feat: support function calls --- examples/postman_echo/debugtalk.py | 4 + .../request_methods/with_functions.yml | 61 ++++++ .../request_methods/with_functions_test.py | 98 +++++++++ httprunner/v3/parser.py | 186 ++++++++++++++++-- httprunner/v3/runner.py | 4 +- httprunner/v3/schema/__init__.py | 3 +- 6 files changed, 338 insertions(+), 18 deletions(-) create mode 100644 examples/postman_echo/request_methods/with_functions.yml create mode 100644 examples/postman_echo/request_methods/with_functions_test.py diff --git a/examples/postman_echo/debugtalk.py b/examples/postman_echo/debugtalk.py index 9ec3149b..849bd537 100644 --- a/examples/postman_echo/debugtalk.py +++ b/examples/postman_echo/debugtalk.py @@ -3,3 +3,7 @@ from httprunner import __version__ def get_httprunner_version(): return __version__ + + +def sum_two(m, n): + return m + n diff --git a/examples/postman_echo/request_methods/with_functions.yml b/examples/postman_echo/request_methods/with_functions.yml new file mode 100644 index 00000000..5391a785 --- /dev/null +++ b/examples/postman_echo/request_methods/with_functions.yml @@ -0,0 +1,61 @@ +config: + name: "request methods testcase with functions" + variables: + foo1: session_bar1 + base_url: "https://postman-echo.com" + verify: False + +teststeps: +- + name: get with params + variables: + foo1: bar1 + foo2: session_bar2 + sum_v: "${sum_two(1, 2)}" + request: + method: GET + url: /get + params: + foo1: $foo1 + foo2: $foo2 + sum_v: $sum_v + headers: + User-Agent: HttpRunner/${get_httprunner_version()} + extract: + session_foo2: "body.args.foo2" + validate: + - eq: ["status_code", 200] + - eq: ["body.args.foo1", "session_bar1"] + - eq: ["body.args.foo2", "session_bar2"] + - eq: ["body.args.sum_v", "3"] +- + name: post raw text + variables: + foo1: "hello world" + foo3: "$session_foo2" + request: + method: POST + url: /post + headers: + User-Agent: HttpRunner/${get_httprunner_version()} + Content-Type: "text/plain" + data: "This is expected to be sent back as part of response body: $foo1-$foo3." + validate: + - eq: ["status_code", 200] + - eq: ["body.data", "This is expected to be sent back as part of response body: session_bar1-session_bar2."] +- + name: post form data + variables: + foo1: bar1 + foo2: bar2 + request: + method: POST + url: /post + headers: + User-Agent: HttpRunner/${get_httprunner_version()} + Content-Type: "application/x-www-form-urlencoded" + data: "foo1=$foo1&foo2=$foo2" + validate: + - eq: ["status_code", 200] + - eq: ["body.form.foo1", "session_bar1"] + - eq: ["body.form.foo2", "bar2"] diff --git a/examples/postman_echo/request_methods/with_functions_test.py b/examples/postman_echo/request_methods/with_functions_test.py new file mode 100644 index 00000000..b8bd9126 --- /dev/null +++ b/examples/postman_echo/request_methods/with_functions_test.py @@ -0,0 +1,98 @@ +from httprunner.v3.runner import TestCaseRunner +from httprunner.v3.schema import TestsConfig, TestStep +from examples.postman_echo import debugtalk + + +class TestCaseRequestMethodsWithFunctions(TestCaseRunner): + config = TestsConfig(**{ + "name": "request methods testcase with functions", + "variables": { + "foo1": "session_bar1" + }, + "functions": { + "get_httprunner_version": debugtalk.get_httprunner_version, + "sum_two": debugtalk.sum_two + }, + "base_url": "https://postman-echo.com", + "verify": False + }) + + teststeps = [ + TestStep(**{ + "name": "get with params", + "variables": { + "foo1": "bar1", + "foo2": "session_bar2", + "sum_v": "${sum_two(1, 2)}" + }, + "request": { + "method": "GET", + "url": "/get", + "params": { + "foo1": "$foo1", + "foo2": "$foo2", + "sum_v": "$sum_v" + }, + "headers": { + "User-Agent": "HttpRunner/${get_httprunner_version()}" + } + }, + "extract": { + "session_foo2": "body.args.foo2" + }, + "validate": [ + {"eq": ["status_code", 200]}, + {"eq": ["body.args.foo1", "session_bar1"]}, + {"eq": ["body.args.foo2", "session_bar2"]}, + {"eq": ["body.args.sum_v", "3"]} + ] + }), + TestStep(**{ + "name": "post raw text", + "variables": { + "foo1": "hello world", + "foo3": "$session_foo2" + }, + "request": { + "method": "POST", + "url": "/post", + "data": "This is expected to be sent back as part of response body: $foo1-$foo3.", + "headers": { + "User-Agent": "HttpRunner/${get_httprunner_version()}", + "Content-Type": "text/plain" + } + }, + "validate": [ + {"eq": ["status_code", 200]}, + {"eq": [ + "body.data", + "This is expected to be sent back as part of response body: session_bar1-session_bar2." + ]}, + ] + }), + TestStep(**{ + "name": "post form data", + "variables": { + "foo1": "session_bar1", + "foo2": "bar2" + }, + "request": { + "method": "POST", + "url": "/post", + "data": "foo1=$foo1&foo2=$foo2", + "headers": { + "User-Agent": "HttpRunner/${get_httprunner_version()}", + "Content-Type": "application/x-www-form-urlencoded" + } + }, + "validate": [ + {"eq": ["status_code", 200]}, + {"eq": ["body.form.foo1", "session_bar1"]}, + {"eq": ["body.form.foo2", "bar2"]} + ] + }) + ] + + +if __name__ == '__main__': + TestCaseRequestMethodsWithFunctions().run() diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index f6ff3a58..54a7f349 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -1,9 +1,9 @@ +import ast import re -from typing import Any, Set, Text -from typing import Dict +from typing import Any, Set, Text, Callable, Tuple, List, Dict, Union from httprunner.v3 import exceptions -from httprunner.v3.exceptions import VariableNotFound +from httprunner.v3.exceptions import VariableNotFound, FunctionNotFound absolute_http_url_regexp = re.compile(r"^https?://", re.I) @@ -15,6 +15,22 @@ variable_regex_compile = re.compile(r"\$\{(\w+)\}|\$(\w+)") function_regex_compile = re.compile(r"\$\{(\w+)\(([\$\w\.\-/\s=,]*)\)\}") +def parse_string_value(str_value: Text) -> Any: + """ parse string to number if possible + e.g. "123" => 123 + "12.2" => 12.3 + "abc" => "abc" + "$var" => "$var" + """ + try: + return ast.literal_eval(str_value) + except ValueError: + return str_value + except SyntaxError: + # e.g. $var, ${func} + return str_value + + def build_url(base_url, path): """ prepend url with base_url unless it's already an absolute URL """ if absolute_http_url_regexp.match(path): @@ -25,7 +41,7 @@ def build_url(base_url, path): raise exceptions.ParamsError("base url missed!") -def regex_findall_variables(content): +def regex_findall_variables(content: Text) -> List[Text]: """ extract all variable names from content, which is in format $variable Args: @@ -59,6 +75,88 @@ def regex_findall_variables(content): return [] +def regex_findall_functions(content: Text) -> List[Text]: + """ extract all functions from string content, which are in format ${fun()} + + Args: + content (str): string content + + Returns: + list: functions list extracted from string content + + Examples: + >>> regex_findall_functions("${func(5)}") + ["func(5)"] + + >>> regex_findall_functions("${func(a=1, b=2)}") + ["func(a=1, b=2)"] + + >>> regex_findall_functions("/api/1000?_t=${get_timestamp()}") + ["get_timestamp()"] + + >>> regex_findall_functions("/api/${add(1, 2)}") + ["add(1, 2)"] + + >>> regex_findall_functions("/api/${add(1, 2)}?_t=${get_timestamp()}") + ["add(1, 2)", "get_timestamp()"] + + """ + try: + return function_regex_compile.findall(content) + except TypeError: + return [] + + +def parse_args_str(arg_str: Text) -> Tuple[List, Dict]: + """ parse function args and kwargs from function. + + Args: + arg_str (str): function str contains args and kwargs + + Returns: + dict: function meta dict + + { + "func_name": "xxx", + "args": [], + "kwargs": {} + } + + Examples: + >>> parse_args_str("") + {'args': [], 'kwargs': {}} + + >>> parse_args_str("5") + {'args': [5], 'kwargs': {}} + + >>> parse_args_str("1, 2") + {'args': [1, 2], 'kwargs': {}} + + >>> parse_args_str("a=1, b=2") + {'args': [], 'kwargs': {'a': 1, 'b': 2}} + + >>> parse_args_str("1, 2, a=3, b=4") + {'args': [1, 2], 'kwargs': {'a':3, 'b':4}} + + """ + args = [] + kwargs = {} + arg_str = arg_str.strip() + if arg_str == "": + return args, kwargs + + arg_list = arg_str.split(',') + for arg in arg_list: + arg = arg.strip() + if '=' in arg: + key, value = arg.split('=') + kwargs[key.strip()] = parse_string_value(value.strip()) + else: + args.append(parse_string_value(arg)) + + return args, kwargs + + def extract_variables(content: Any) -> Set: """ extract all variables in content recursively. """ @@ -80,7 +178,59 @@ def extract_variables(content: Any) -> Set: return set() -def parse_string_variables(content, variables_mapping): +def parse_string_functions( + content: Text, + variables_mapping: Dict[Text, Any], + functions_mapping: Dict[Text, Callable]) -> Text: + """ parse string content with functions mapping. + + Args: + content (str): string content to be parsed. + variables_mapping (dict): variables mapping. + functions_mapping (dict): functions mapping. + + Returns: + str: parsed string content. + + Examples: + >>> content = "abc${add_one(3)}def" + >>> functions_mapping = {"add_one": lambda x: x + 1} + >>> parse_string_functions(content, {}, functions_mapping) + "abc4def" + + """ + functions_list = regex_findall_functions(content) + for func_meta_tuple in functions_list: + func_name, args_str = func_meta_tuple + args, kwargs = parse_args_str(args_str) + + args = parse_content(args, variables_mapping, functions_mapping) + kwargs = parse_content(kwargs, variables_mapping, functions_mapping) + + try: + func = functions_mapping[func_name] + except KeyError: + raise FunctionNotFound(f"{func_name} not found in {functions_mapping}") + + eval_value = func(*args, **kwargs) + + func_content = "${" + func_name + f"({args_str})" + "}" + if func_content == content: + # content is a function, e.g. "${add_one(3)}" + content = eval_value + else: + # content contains one or many functions, e.g. "abc${add_one(3)}def" + content = content.replace( + func_content, + str(eval_value), 1 + ) + + return content + + +def parse_string_variables( + content: Text, + variables_mapping: Dict[Text, Any]) -> Text: """ parse string content with variables mapping. Args: @@ -92,7 +242,7 @@ def parse_string_variables(content, variables_mapping): Examples: >>> content = "/api/users/$uid" - >>> variables_mapping = {"$uid": 1000} + >>> variables_mapping = {"uid": 1000} >>> parse_string_variables(content, variables_mapping) "/api/users/1000" @@ -102,7 +252,7 @@ def parse_string_variables(content, variables_mapping): try: variable_value = variables_mapping[variable_name] except KeyError: - raise VariableNotFound(f"{variable_name} not in {variables_mapping}") + raise VariableNotFound(f"{variable_name} not found in {variables_mapping}") # TODO: replace variable label from $var to {{var}} if f"${variable_name}" == content: @@ -121,7 +271,10 @@ def parse_string_variables(content, variables_mapping): return content -def parse_content(content: Any, variables_mapping: Dict[str, Any] = None, functions_mapping=None): +def parse_content( + content: Any, + variables_mapping: Dict[Text, Any] = None, + functions_mapping: Dict[Text, Callable] = None) -> Any: """ parse content with evaluated variables mapping. Notice: variables_mapping should not contain any variable or function. """ @@ -136,8 +289,8 @@ def parse_content(content: Any, variables_mapping: Dict[str, Any] = None, functi content = content.strip() # replace functions with evaluated value - # Notice: _eval_content_functions must be called before _eval_content_variables - # content = parse_string_functions(content, variables_mapping, functions_mapping) + # Notice: parse_string_functions must be called before parse_string_variables + content = parse_string_functions(content, variables_mapping, functions_mapping) # replace variables with binding value content = parse_string_variables(content, variables_mapping) @@ -146,15 +299,15 @@ def parse_content(content: Any, variables_mapping: Dict[str, Any] = None, functi elif isinstance(content, (list, set, tuple)): return [ - parse_content(item, variables_mapping) + parse_content(item, variables_mapping, functions_mapping) for item in content ] elif isinstance(content, dict): parsed_content = {} for key, value in content.items(): - parsed_key = parse_content(key, variables_mapping) - parsed_value = parse_content(value, variables_mapping) + parsed_key = parse_content(key, variables_mapping, functions_mapping) + parsed_value = parse_content(value, variables_mapping, functions_mapping) parsed_content[parsed_key] = parsed_value return parsed_content @@ -162,7 +315,9 @@ def parse_content(content: Any, variables_mapping: Dict[str, Any] = None, functi return content -def parse_variables_mapping(variables_mapping: Dict[Text, Any]) -> Dict[Text, Any]: +def parse_variables_mapping( + variables_mapping: Dict[Text, Any], + functions_mapping: Dict[Text, Callable] = None) -> Dict[Text, Any]: parsed_variables: Dict[Text, Any] = {} @@ -194,7 +349,8 @@ def parse_variables_mapping(variables_mapping: Dict[Text, Any]) -> Dict[Text, An raise VariableNotFound(not_defined_variables) try: - parsed_value = parse_content(var_value, parsed_variables) + parsed_value = parse_content( + var_value, parsed_variables, functions_mapping) except VariableNotFound: continue diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 12d28059..76277225 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -27,7 +27,7 @@ class TestCaseRunner(object): # parse request_dict = step.request.dict() - parsed_request_dict = parse_content(request_dict, step.variables) + parsed_request_dict = parse_content(request_dict, step.variables, self.config.functions) # prepare arguments method = parsed_request_dict.pop("method") @@ -62,7 +62,7 @@ class TestCaseRunner(object): # update with session variables extracted from former step step.variables.update(session_variables) # parse variables - step.variables = parse_variables_mapping(step.variables) + step.variables = parse_variables_mapping(step.variables, self.config.functions) # run step extract_mapping = self.run_step(step) # save extracted variables to session variables diff --git a/httprunner/v3/schema/__init__.py b/httprunner/v3/schema/__init__.py index 778a217a..ea1220d2 100644 --- a/httprunner/v3/schema/__init__.py +++ b/httprunner/v3/schema/__init__.py @@ -1,6 +1,6 @@ from enum import Enum from typing import Any -from typing import Dict, List, Text, Union +from typing import Dict, List, Text, Union, Callable from pydantic import BaseModel, Field from pydantic import HttpUrl @@ -34,6 +34,7 @@ class TestsConfig(BaseModel): verify: Verify = False base_url: BaseUrl = "" variables: Variables = {} + functions: Dict[Text, Callable] setup_hooks: Hook = [] teardown_hooks: Hook = [] export: Export = [] From 12148a13063fe81b1c5122d8b1c162f01ba7bc92 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 20 Apr 2020 18:40:36 +0800 Subject: [PATCH 11/49] test: add unittest for v3 parser --- httprunner/v3/parser_test.py | 308 ++++++++++++++++++++++++++++++++++- 1 file changed, 304 insertions(+), 4 deletions(-) diff --git a/httprunner/v3/parser_test.py b/httprunner/v3/parser_test.py index f4a02068..51b067f4 100644 --- a/httprunner/v3/parser_test.py +++ b/httprunner/v3/parser_test.py @@ -1,7 +1,8 @@ +import time import unittest -from httprunner.v3.parser import parse_variables_mapping -from httprunner.v3.exceptions import VariableNotFound +from httprunner.v3 import parser +from httprunner.v3.exceptions import VariableNotFound, FunctionNotFound class TestParserBasic(unittest.TestCase): @@ -14,7 +15,7 @@ class TestParserBasic(unittest.TestCase): "a": 1, "b": 2 } - parsed_variables = parse_variables_mapping(variables) + parsed_variables = parser.parse_variables_mapping(variables) print(parsed_variables) self.assertEqual(parsed_variables["varA"], "123") self.assertEqual(parsed_variables["varB"], "123") @@ -27,4 +28,303 @@ class TestParserBasic(unittest.TestCase): "b": 2 } with self.assertRaises(VariableNotFound): - parse_variables_mapping(variables) + parser.parse_variables_mapping(variables) + + def test_parse_string_value(self): + self.assertEqual(parser.parse_string_value("123"), 123) + self.assertEqual(parser.parse_string_value("12.3"), 12.3) + self.assertEqual(parser.parse_string_value("a123"), "a123") + self.assertEqual(parser.parse_string_value("$var"), "$var") + self.assertEqual(parser.parse_string_value("${func}"), "${func}") + + def test_extract_variables(self): + self.assertEqual( + parser.extract_variables("$var"), + {"var"} + ) + self.assertEqual( + parser.extract_variables("$var123"), + {"var123"} + ) + self.assertEqual( + parser.extract_variables("$var_name"), + {"var_name"} + ) + self.assertEqual( + parser.extract_variables("var"), + set() + ) + self.assertEqual( + parser.extract_variables("a$var"), + {"var"} + ) + self.assertEqual( + parser.extract_variables("$v ar"), + {"v"} + ) + self.assertEqual( + parser.extract_variables(" "), + set() + ) + self.assertEqual( + parser.extract_variables("$abc*"), + {"abc"} + ) + self.assertEqual( + parser.extract_variables("${func()}"), + set() + ) + self.assertEqual( + parser.extract_variables("${func(1,2)}"), + set() + ) + self.assertEqual( + parser.extract_variables("${gen_md5($TOKEN, $data, $random)}"), + {"TOKEN", "data", "random"} + ) + + def test_parse_function(self): + self.assertEqual( + parser.parse_args_str(""), + ([], {}) + ) + self.assertEqual( + parser.parse_args_str("5"), + ([5], {}) + ) + self.assertEqual( + parser.parse_args_str("1, 2"), + ([1, 2], {}) + ) + self.assertEqual( + parser.parse_args_str("a=1, b=2"), + ([], {'a': 1, 'b': 2}) + ) + self.assertEqual( + parser.parse_args_str("a= 1, b =2"), + ([], {'a': 1, 'b': 2}) + ) + self.assertEqual( + parser.parse_args_str("1, 2, a=3, b=4"), + ([1, 2], {'a': 3, 'b': 4}) + ) + self.assertEqual( + parser.parse_args_str("$request, 123"), + (["$request", 123], {}) + ) + self.assertEqual( + parser.parse_args_str(" "), + ([], {}) + ) + self.assertEqual( + parser.parse_args_str("hello world, a=3, b=4"), + (["hello world"], {'a': 3, 'b': 4}) + ) + self.assertEqual( + parser.parse_args_str("$request, 12 3"), + (["$request", '12 3'], {}) + ) + + def test_extract_functions(self): + self.assertEqual( + parser.regex_findall_functions("${func()}"), + [("func", "")] + ) + self.assertEqual( + parser.regex_findall_functions("${func(5)}"), + [("func", "5")] + ) + self.assertEqual( + parser.regex_findall_functions("${func(a=1, b=2)}"), + [("func", "a=1, b=2")] + ) + self.assertEqual( + parser.regex_findall_functions("${func(1, $b, c=$x, d=4)}"), + [("func", "1, $b, c=$x, d=4")] + ) + self.assertEqual( + parser.regex_findall_functions("/api/1000?_t=${get_timestamp()}"), + [("get_timestamp", "")] + ) + self.assertEqual( + parser.regex_findall_functions("/api/${add(1, 2)}"), + [("add", "1, 2")] + ) + self.assertEqual( + parser.regex_findall_functions("/api/${add(1, 2)}?_t=${get_timestamp()}"), + [('add', '1, 2'), ('get_timestamp', '')] + ) + self.assertEqual( + parser.regex_findall_functions("abc${func(1, 2, a=3, b=4)}def"), + [('func', '1, 2, a=3, b=4')] + ) + + def test_parse_content(self): + content = { + 'request': { + 'url': '/api/users/$uid', + 'method': "$method", + 'headers': {'token': '$token'}, + 'data': { + "null": None, + "true": True, + "false": False, + "empty_str": "", + "value": "abc${add_one(3)}def" + } + } + } + variables_mapping = { + "uid": 1000, + "method": "POST", + "token": "abc123" + } + functions_mapping = { + "add_one": lambda x: x + 1 + } + result = parser.parse_content(content, variables_mapping, functions_mapping) + self.assertEqual("/api/users/1000", result["request"]["url"]) + self.assertEqual("abc123", result["request"]["headers"]["token"]) + self.assertEqual("POST", result["request"]["method"]) + self.assertIsNone(result["request"]["data"]["null"]) + self.assertTrue(result["request"]["data"]["true"]) + self.assertFalse(result["request"]["data"]["false"]) + self.assertEqual("", result["request"]["data"]["empty_str"]) + self.assertEqual("abc4def", result["request"]["data"]["value"]) + + def test_parse_data_variables(self): + variables_mapping = { + "var_1": "abc", + "var_2": "def", + "var_3": 123, + "var_4": {"a": 1}, + "var_5": True, + "var_6": None + } + self.assertEqual( + parser.parse_content("$var_1", variables_mapping), + "abc" + ) + self.assertEqual( + parser.parse_content("var_1", variables_mapping), + "var_1" + ) + self.assertEqual( + parser.parse_content("$var_1#XYZ", variables_mapping), + "abc#XYZ" + ) + self.assertEqual( + parser.parse_content("/$var_1/$var_2/var3", variables_mapping), + "/abc/def/var3" + ) + self.assertEqual( + parser.parse_string_variables("${func($var_1, $var_2, xyz)}", variables_mapping), + "${func(abc, def, xyz)}" + ) + self.assertEqual( + parser.parse_content("$var_3", variables_mapping), + 123 + ) + self.assertEqual( + parser.parse_content("$var_4", variables_mapping), + {"a": 1} + ) + self.assertEqual( + parser.parse_content("$var_5", variables_mapping), + True + ) + self.assertEqual( + parser.parse_content("abc$var_5", variables_mapping), + "abcTrue" + ) + self.assertEqual( + parser.parse_content("abc$var_4", variables_mapping), + "abc{'a': 1}" + ) + self.assertEqual( + parser.parse_content("$var_6", variables_mapping), + None + ) + + with self.assertRaises(VariableNotFound): + parser.parse_content("/api/$SECRET_KEY", variables_mapping) + + self.assertEqual( + parser.parse_content(["$var_1", "$var_2"], variables_mapping), + ["abc", "def"] + ) + self.assertEqual( + parser.parse_content({"$var_1": "$var_2"}, variables_mapping), + {"abc": "def"} + ) + + def test_parse_data_functions(self): + import random, string + functions_mapping = { + "gen_random_string": lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) \ + for _ in range(str_len)) + } + result = parser.parse_content("${gen_random_string(5)}", functions_mapping=functions_mapping) + self.assertEqual(len(result), 5) + + add_two_nums = lambda a, b=1: a + b + functions_mapping["add_two_nums"] = add_two_nums + self.assertEqual( + parser.parse_content("${add_two_nums(1)}", functions_mapping=functions_mapping), + 2 + ) + self.assertEqual( + parser.parse_content("${add_two_nums(1, 2)}", functions_mapping=functions_mapping), + 3 + ) + self.assertEqual( + parser.parse_content("/api/${add_two_nums(1, 2)}", functions_mapping=functions_mapping), + "/api/3" + ) + + with self.assertRaises(FunctionNotFound): + parser.parse_content("/api/${gen_md5(abc)}") + + def test_parse_data_testcase(self): + variables = { + "uid": "1000", + "random": "A2dEx", + "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d", + "data": {"name": "user", "password": "123456"} + } + functions = { + "add_two_nums": lambda a, b=1: a + b, + "get_timestamp": lambda: int(time.time() * 1000) + } + testcase_template = { + "url": "http://127.0.0.1:5000/api/users/$uid/${add_two_nums(1,2)}", + "method": "POST", + "headers": { + "Content-Type": "application/json", + "authorization": "$authorization", + "random": "$random", + "sum": "${add_two_nums(1, 2)}" + }, + "body": "$data" + } + parsed_testcase = parser.parse_content(testcase_template, variables, functions) + self.assertEqual( + parsed_testcase["url"], + "http://127.0.0.1:5000/api/users/1000/3" + ) + self.assertEqual( + parsed_testcase["headers"]["authorization"], + variables["authorization"] + ) + self.assertEqual( + parsed_testcase["headers"]["random"], + variables["random"] + ) + self.assertEqual( + parsed_testcase["body"], + variables["data"] + ) + self.assertEqual( + parsed_testcase["headers"]["sum"], + 3 + ) From 8c625f08f348e5f411728d880e61b8a419c17933 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 11:38:33 +0800 Subject: [PATCH 12/49] refactor: implement parse_string to eval string with both variables and functions --- httprunner/v3/parser.py | 277 ++++++++++++++++++----------------- httprunner/v3/parser_test.py | 87 +++++++---- 2 files changed, 203 insertions(+), 161 deletions(-) diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index 54a7f349..965d6f22 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -107,56 +107,6 @@ def regex_findall_functions(content: Text) -> List[Text]: return [] -def parse_args_str(arg_str: Text) -> Tuple[List, Dict]: - """ parse function args and kwargs from function. - - Args: - arg_str (str): function str contains args and kwargs - - Returns: - dict: function meta dict - - { - "func_name": "xxx", - "args": [], - "kwargs": {} - } - - Examples: - >>> parse_args_str("") - {'args': [], 'kwargs': {}} - - >>> parse_args_str("5") - {'args': [5], 'kwargs': {}} - - >>> parse_args_str("1, 2") - {'args': [1, 2], 'kwargs': {}} - - >>> parse_args_str("a=1, b=2") - {'args': [], 'kwargs': {'a': 1, 'b': 2}} - - >>> parse_args_str("1, 2, a=3, b=4") - {'args': [1, 2], 'kwargs': {'a':3, 'b':4}} - - """ - args = [] - kwargs = {} - arg_str = arg_str.strip() - if arg_str == "": - return args, kwargs - - arg_list = arg_str.split(',') - for arg in arg_list: - arg = arg.strip() - if '=' in arg: - key, value = arg.split('=') - kwargs[key.strip()] = parse_string_value(value.strip()) - else: - args.append(parse_string_value(arg)) - - return args, kwargs - - def extract_variables(content: Any) -> Set: """ extract all variables in content recursively. """ @@ -178,97 +128,156 @@ def extract_variables(content: Any) -> Set: return set() -def parse_string_functions( - content: Text, - variables_mapping: Dict[Text, Any], - functions_mapping: Dict[Text, Callable]) -> Text: - """ parse string content with functions mapping. +def parse_function_params(params): + """ parse function params to args and kwargs. Args: - content (str): string content to be parsed. - variables_mapping (dict): variables mapping. - functions_mapping (dict): functions mapping. + params (str): function param in string + + Returns: + dict: function meta dict + + { + "args": [], + "kwargs": {} + } + + Examples: + >>> parse_function_params("") + {'args': [], 'kwargs': {}} + + >>> parse_function_params("5") + {'args': [5], 'kwargs': {}} + + >>> parse_function_params("1, 2") + {'args': [1, 2], 'kwargs': {}} + + >>> parse_function_params("a=1, b=2") + {'args': [], 'kwargs': {'a': 1, 'b': 2}} + + >>> parse_function_params("1, 2, a=3, b=4") + {'args': [1, 2], 'kwargs': {'a':3, 'b':4}} + + """ + function_meta = { + "args": [], + "kwargs": {} + } + + params_str = params.strip() + if params_str == "": + return function_meta + + args_list = params_str.split(',') + for arg in args_list: + arg = arg.strip() + if '=' in arg: + key, value = arg.split('=') + function_meta["kwargs"][key.strip()] = parse_string_value(value.strip()) + else: + function_meta["args"].append(parse_string_value(arg)) + + return function_meta + + +def parse_string( + raw_string: Text, + variables_mapping: Dict[Text, Any], + functions_mapping: Dict[Text, Callable]) -> Text: + """ parse string content with variables and functions mapping. + + Args: + raw_string: raw string content to be parsed. + variables_mapping: variables mapping. + functions_mapping: functions mapping. Returns: str: parsed string content. Examples: - >>> content = "abc${add_one(3)}def" + >>> raw_string = "abc${add_one($num)}def" + >>> variables_mapping = {"num": 3} >>> functions_mapping = {"add_one": lambda x: x + 1} - >>> parse_string_functions(content, {}, functions_mapping) + >>> parse_string(raw_string, variables_mapping, functions_mapping) "abc4def" """ - functions_list = regex_findall_functions(content) - for func_meta_tuple in functions_list: - func_name, args_str = func_meta_tuple - args, kwargs = parse_args_str(args_str) + try: + match_start_position = raw_string.index("$", 0) + parsed_string = raw_string[0:match_start_position] + except ValueError: + parsed_string = raw_string + return parsed_string - args = parse_content(args, variables_mapping, functions_mapping) - kwargs = parse_content(kwargs, variables_mapping, functions_mapping) + while match_start_position < len(raw_string): + # Notice: notation priority + # $$ > ${func($a, $b)} > $var + + # search $$ + dollar_match = dolloar_regex_compile.match(raw_string, match_start_position) + if dollar_match: + match_start_position = dollar_match.end() + parsed_string += "$" + continue + + # search function like ${func($a, $b)} + func_match = function_regex_compile.match(raw_string, match_start_position) + if func_match: + func_name = func_match.group(1) + try: + func = functions_mapping[func_name] + except KeyError: + raise FunctionNotFound(f"{func_name} not found in {functions_mapping}") + + func_params_str = func_match.group(2) + function_meta = parse_function_params(func_params_str) + args = function_meta["args"] + kwargs = function_meta["kwargs"] + func_eval_value = func(*args, **kwargs) + + func_raw_str = "${" + func_name + f"({func_params_str})" + "}" + if func_raw_str == raw_string: + # raw_string is a function, e.g. "${add_one(3)}", return its eval value directly + return func_eval_value + + # raw_string contains one or many functions, e.g. "abc${add_one(3)}def" + parsed_string += str(func_eval_value) + match_start_position = func_match.end() + continue + + # search variable like ${var} or $var + var_match = variable_regex_compile.match(raw_string, match_start_position) + if var_match: + var_name = var_match.group(1) or var_match.group(2) + # check if any variable undefined in variables_mapping + try: + var_value = variables_mapping[var_name] + except KeyError: + raise VariableNotFound(f"{var_name} not found in {variables_mapping}") + + if f"${var_name}" == raw_string or "${" + var_name + "}" == raw_string: + # raw_string is a variable, $var or ${var}, return its value directly + return var_value + + # raw_string contains one or many variables, e.g. "abc${var}def" + parsed_string += str(var_value) + match_start_position = var_match.end() + continue + + curr_position = match_start_position try: - func = functions_mapping[func_name] - except KeyError: - raise FunctionNotFound(f"{func_name} not found in {functions_mapping}") + # find next $ location + match_start_position = raw_string.index("$", curr_position + 1) + remain_string = raw_string[curr_position:match_start_position] + except ValueError: + remain_string = raw_string[curr_position:] + # break while loop + match_start_position = len(raw_string) - eval_value = func(*args, **kwargs) + parsed_string += remain_string - func_content = "${" + func_name + f"({args_str})" + "}" - if func_content == content: - # content is a function, e.g. "${add_one(3)}" - content = eval_value - else: - # content contains one or many functions, e.g. "abc${add_one(3)}def" - content = content.replace( - func_content, - str(eval_value), 1 - ) - - return content - - -def parse_string_variables( - content: Text, - variables_mapping: Dict[Text, Any]) -> Text: - """ parse string content with variables mapping. - - Args: - content (str): string content to be parsed. - variables_mapping (dict): variables mapping. - - Returns: - str: parsed string content. - - Examples: - >>> content = "/api/users/$uid" - >>> variables_mapping = {"uid": 1000} - >>> parse_string_variables(content, variables_mapping) - "/api/users/1000" - - """ - variables_list = extract_variables(content) - for variable_name in variables_list: - try: - variable_value = variables_mapping[variable_name] - except KeyError: - raise VariableNotFound(f"{variable_name} not found in {variables_mapping}") - - # TODO: replace variable label from $var to {{var}} - if f"${variable_name}" == content: - # content is a variable - content = variable_value - else: - # content contains one or several variables - if not isinstance(variable_value, str): - variable_value = str(variable_value) - - content = content.replace( - f"${variable_name}", - variable_value, 1 - ) - - return content + return parsed_string def parse_content( @@ -278,24 +287,20 @@ def parse_content( """ parse content with evaluated variables mapping. Notice: variables_mapping should not contain any variable or function. """ - # TODO: refactor type check - if content is None or isinstance(content, (int, float, bool)): - return content - - elif isinstance(content, str): - # content is in string format here + if isinstance(content, str): + # content in string format may contains variables and functions variables_mapping = variables_mapping or {} functions_mapping = functions_mapping or {} content = content.strip() # replace functions with evaluated value # Notice: parse_string_functions must be called before parse_string_variables - content = parse_string_functions(content, variables_mapping, functions_mapping) + # content = parse_string_functions(content, variables_mapping, functions_mapping) # replace variables with binding value - content = parse_string_variables(content, variables_mapping) + # content = parse_string_variables(content, variables_mapping) - return content + return parse_string(content, variables_mapping, functions_mapping) elif isinstance(content, (list, set, tuple)): return [ @@ -312,7 +317,9 @@ def parse_content( return parsed_content - return content + else: + # other types, e.g. None, int, float, bool + return content def parse_variables_mapping( diff --git a/httprunner/v3/parser_test.py b/httprunner/v3/parser_test.py index 51b067f4..32546c2b 100644 --- a/httprunner/v3/parser_test.py +++ b/httprunner/v3/parser_test.py @@ -83,46 +83,46 @@ class TestParserBasic(unittest.TestCase): {"TOKEN", "data", "random"} ) - def test_parse_function(self): + def test_parse_function_params(self): self.assertEqual( - parser.parse_args_str(""), - ([], {}) + parser.parse_function_params(""), + {'args': [], 'kwargs': {}} ) self.assertEqual( - parser.parse_args_str("5"), - ([5], {}) + parser.parse_function_params("5"), + {'args': [5], 'kwargs': {}} ) self.assertEqual( - parser.parse_args_str("1, 2"), - ([1, 2], {}) + parser.parse_function_params("1, 2"), + {'args': [1, 2], 'kwargs': {}} ) self.assertEqual( - parser.parse_args_str("a=1, b=2"), - ([], {'a': 1, 'b': 2}) + parser.parse_function_params("a=1, b=2"), + {'args': [], 'kwargs': {'a': 1, 'b': 2}} ) self.assertEqual( - parser.parse_args_str("a= 1, b =2"), - ([], {'a': 1, 'b': 2}) + parser.parse_function_params("a= 1, b =2"), + {'args': [], 'kwargs': {'a': 1, 'b': 2}} ) self.assertEqual( - parser.parse_args_str("1, 2, a=3, b=4"), - ([1, 2], {'a': 3, 'b': 4}) + parser.parse_function_params("1, 2, a=3, b=4"), + {'args': [1, 2], 'kwargs': {'a': 3, 'b': 4}} ) self.assertEqual( - parser.parse_args_str("$request, 123"), - (["$request", 123], {}) + parser.parse_function_params("$request, 123"), + {'args': ["$request", 123], 'kwargs': {}} ) self.assertEqual( - parser.parse_args_str(" "), - ([], {}) + parser.parse_function_params(" "), + {'args': [], 'kwargs': {}} ) self.assertEqual( - parser.parse_args_str("hello world, a=3, b=4"), - (["hello world"], {'a': 3, 'b': 4}) + parser.parse_function_params("hello world, a=3, b=4"), + {'args': ["hello world"], 'kwargs': {'a': 3, 'b': 4}} ) self.assertEqual( - parser.parse_args_str("$request, 12 3"), - (["$request", '12 3'], {}) + parser.parse_function_params("$request, 12 3"), + {'args': ["$request", '12 3'], 'kwargs': {}} ) def test_extract_functions(self): @@ -192,7 +192,7 @@ class TestParserBasic(unittest.TestCase): self.assertEqual("", result["request"]["data"]["empty_str"]) self.assertEqual("abc4def", result["request"]["data"]["value"]) - def test_parse_data_variables(self): + def test_parse_content_with_variables(self): variables_mapping = { "var_1": "abc", "var_2": "def", @@ -205,6 +205,10 @@ class TestParserBasic(unittest.TestCase): parser.parse_content("$var_1", variables_mapping), "abc" ) + self.assertEqual( + parser.parse_content("${var_1}", variables_mapping), + "abc" + ) self.assertEqual( parser.parse_content("var_1", variables_mapping), "var_1" @@ -214,12 +218,12 @@ class TestParserBasic(unittest.TestCase): "abc#XYZ" ) self.assertEqual( - parser.parse_content("/$var_1/$var_2/var3", variables_mapping), - "/abc/def/var3" + parser.parse_content("${var_1}#XYZ", variables_mapping), + "abc#XYZ" ) self.assertEqual( - parser.parse_string_variables("${func($var_1, $var_2, xyz)}", variables_mapping), - "${func(abc, def, xyz)}" + parser.parse_content("/$var_1/$var_2/var3", variables_mapping), + "/abc/def/var3" ) self.assertEqual( parser.parse_content("$var_3", variables_mapping), @@ -258,6 +262,37 @@ class TestParserBasic(unittest.TestCase): {"abc": "def"} ) + def test_parse_data_multiple_identical_variables(self): + variables_mapping = { + "var_1": "abc", + "var_2": "def", + } + self.assertEqual( + parser.parse_content("/$var_1/$var_2/$var_1", variables_mapping), + "/abc/def/abc" + ) + + variables_mapping = { + "userid": 100, + "data": 1498 + } + content = "/users/$userid/training/$data?userId=$userid&data=$data" + self.assertEqual( + parser.parse_content(content, variables_mapping), + "/users/100/training/1498?userId=100&data=1498" + ) + + variables_mapping = { + "user": 100, + "userid": 1000, + "data": 1498 + } + content = "/users/$user/$userid/$data?userId=$userid&data=$data" + self.assertEqual( + parser.parse_content(content, variables_mapping), + "/users/100/1000/1498?userId=1000&data=1498" + ) + def test_parse_data_functions(self): import random, string functions_mapping = { From cf3653cf18b67b6af9cc01de9420c763e3889abe Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 11:44:00 +0800 Subject: [PATCH 13/49] refactor: change function name from parse_content to parse_data --- httprunner/v3/parser.py | 44 +++++++++++++------------------ httprunner/v3/parser_test.py | 50 ++++++++++++++++++------------------ httprunner/v3/runner.py | 4 +-- 3 files changed, 45 insertions(+), 53 deletions(-) diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index 965d6f22..8292bb36 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -280,46 +280,38 @@ def parse_string( return parsed_string -def parse_content( - content: Any, +def parse_data( + raw_data: Any, variables_mapping: Dict[Text, Any] = None, functions_mapping: Dict[Text, Callable] = None) -> Any: - """ parse content with evaluated variables mapping. + """ parse raw data with evaluated variables mapping. Notice: variables_mapping should not contain any variable or function. """ - if isinstance(content, str): + if isinstance(raw_data, str): # content in string format may contains variables and functions variables_mapping = variables_mapping or {} functions_mapping = functions_mapping or {} - content = content.strip() + raw_data = raw_data.strip() + return parse_string(raw_data, variables_mapping, functions_mapping) - # replace functions with evaluated value - # Notice: parse_string_functions must be called before parse_string_variables - # content = parse_string_functions(content, variables_mapping, functions_mapping) - - # replace variables with binding value - # content = parse_string_variables(content, variables_mapping) - - return parse_string(content, variables_mapping, functions_mapping) - - elif isinstance(content, (list, set, tuple)): + elif isinstance(raw_data, (list, set, tuple)): return [ - parse_content(item, variables_mapping, functions_mapping) - for item in content + parse_data(item, variables_mapping, functions_mapping) + for item in raw_data ] - elif isinstance(content, dict): - parsed_content = {} - for key, value in content.items(): - parsed_key = parse_content(key, variables_mapping, functions_mapping) - parsed_value = parse_content(value, variables_mapping, functions_mapping) - parsed_content[parsed_key] = parsed_value + elif isinstance(raw_data, dict): + parsed_data = {} + for key, value in raw_data.items(): + parsed_key = parse_data(key, variables_mapping, functions_mapping) + parsed_value = parse_data(value, variables_mapping, functions_mapping) + parsed_data[parsed_key] = parsed_value - return parsed_content + return parsed_data else: # other types, e.g. None, int, float, bool - return content + return raw_data def parse_variables_mapping( @@ -356,7 +348,7 @@ def parse_variables_mapping( raise VariableNotFound(not_defined_variables) try: - parsed_value = parse_content( + parsed_value = parse_data( var_value, parsed_variables, functions_mapping) except VariableNotFound: continue diff --git a/httprunner/v3/parser_test.py b/httprunner/v3/parser_test.py index 32546c2b..c63cab93 100644 --- a/httprunner/v3/parser_test.py +++ b/httprunner/v3/parser_test.py @@ -182,7 +182,7 @@ class TestParserBasic(unittest.TestCase): functions_mapping = { "add_one": lambda x: x + 1 } - result = parser.parse_content(content, variables_mapping, functions_mapping) + result = parser.parse_data(content, variables_mapping, functions_mapping) self.assertEqual("/api/users/1000", result["request"]["url"]) self.assertEqual("abc123", result["request"]["headers"]["token"]) self.assertEqual("POST", result["request"]["method"]) @@ -202,63 +202,63 @@ class TestParserBasic(unittest.TestCase): "var_6": None } self.assertEqual( - parser.parse_content("$var_1", variables_mapping), + parser.parse_data("$var_1", variables_mapping), "abc" ) self.assertEqual( - parser.parse_content("${var_1}", variables_mapping), + parser.parse_data("${var_1}", variables_mapping), "abc" ) self.assertEqual( - parser.parse_content("var_1", variables_mapping), + parser.parse_data("var_1", variables_mapping), "var_1" ) self.assertEqual( - parser.parse_content("$var_1#XYZ", variables_mapping), + parser.parse_data("$var_1#XYZ", variables_mapping), "abc#XYZ" ) self.assertEqual( - parser.parse_content("${var_1}#XYZ", variables_mapping), + parser.parse_data("${var_1}#XYZ", variables_mapping), "abc#XYZ" ) self.assertEqual( - parser.parse_content("/$var_1/$var_2/var3", variables_mapping), + parser.parse_data("/$var_1/$var_2/var3", variables_mapping), "/abc/def/var3" ) self.assertEqual( - parser.parse_content("$var_3", variables_mapping), + parser.parse_data("$var_3", variables_mapping), 123 ) self.assertEqual( - parser.parse_content("$var_4", variables_mapping), + parser.parse_data("$var_4", variables_mapping), {"a": 1} ) self.assertEqual( - parser.parse_content("$var_5", variables_mapping), + parser.parse_data("$var_5", variables_mapping), True ) self.assertEqual( - parser.parse_content("abc$var_5", variables_mapping), + parser.parse_data("abc$var_5", variables_mapping), "abcTrue" ) self.assertEqual( - parser.parse_content("abc$var_4", variables_mapping), + parser.parse_data("abc$var_4", variables_mapping), "abc{'a': 1}" ) self.assertEqual( - parser.parse_content("$var_6", variables_mapping), + parser.parse_data("$var_6", variables_mapping), None ) with self.assertRaises(VariableNotFound): - parser.parse_content("/api/$SECRET_KEY", variables_mapping) + parser.parse_data("/api/$SECRET_KEY", variables_mapping) self.assertEqual( - parser.parse_content(["$var_1", "$var_2"], variables_mapping), + parser.parse_data(["$var_1", "$var_2"], variables_mapping), ["abc", "def"] ) self.assertEqual( - parser.parse_content({"$var_1": "$var_2"}, variables_mapping), + parser.parse_data({"$var_1": "$var_2"}, variables_mapping), {"abc": "def"} ) @@ -268,7 +268,7 @@ class TestParserBasic(unittest.TestCase): "var_2": "def", } self.assertEqual( - parser.parse_content("/$var_1/$var_2/$var_1", variables_mapping), + parser.parse_data("/$var_1/$var_2/$var_1", variables_mapping), "/abc/def/abc" ) @@ -278,7 +278,7 @@ class TestParserBasic(unittest.TestCase): } content = "/users/$userid/training/$data?userId=$userid&data=$data" self.assertEqual( - parser.parse_content(content, variables_mapping), + parser.parse_data(content, variables_mapping), "/users/100/training/1498?userId=100&data=1498" ) @@ -289,7 +289,7 @@ class TestParserBasic(unittest.TestCase): } content = "/users/$user/$userid/$data?userId=$userid&data=$data" self.assertEqual( - parser.parse_content(content, variables_mapping), + parser.parse_data(content, variables_mapping), "/users/100/1000/1498?userId=1000&data=1498" ) @@ -299,26 +299,26 @@ class TestParserBasic(unittest.TestCase): "gen_random_string": lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) \ for _ in range(str_len)) } - result = parser.parse_content("${gen_random_string(5)}", functions_mapping=functions_mapping) + result = parser.parse_data("${gen_random_string(5)}", functions_mapping=functions_mapping) self.assertEqual(len(result), 5) add_two_nums = lambda a, b=1: a + b functions_mapping["add_two_nums"] = add_two_nums self.assertEqual( - parser.parse_content("${add_two_nums(1)}", functions_mapping=functions_mapping), + parser.parse_data("${add_two_nums(1)}", functions_mapping=functions_mapping), 2 ) self.assertEqual( - parser.parse_content("${add_two_nums(1, 2)}", functions_mapping=functions_mapping), + parser.parse_data("${add_two_nums(1, 2)}", functions_mapping=functions_mapping), 3 ) self.assertEqual( - parser.parse_content("/api/${add_two_nums(1, 2)}", functions_mapping=functions_mapping), + parser.parse_data("/api/${add_two_nums(1, 2)}", functions_mapping=functions_mapping), "/api/3" ) with self.assertRaises(FunctionNotFound): - parser.parse_content("/api/${gen_md5(abc)}") + parser.parse_data("/api/${gen_md5(abc)}") def test_parse_data_testcase(self): variables = { @@ -342,7 +342,7 @@ class TestParserBasic(unittest.TestCase): }, "body": "$data" } - parsed_testcase = parser.parse_content(testcase_template, variables, functions) + parsed_testcase = parser.parse_data(testcase_template, variables, functions) self.assertEqual( parsed_testcase["url"], "http://127.0.0.1:5000/api/users/1000/3" diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 76277225..3e3c3489 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -3,7 +3,7 @@ from typing import List import requests from loguru import logger -from httprunner.v3.parser import build_url, parse_content, parse_variables_mapping +from httprunner.v3.parser import build_url, parse_data, parse_variables_mapping from httprunner.v3.response import ResponseObject from httprunner.v3.schema import TestsConfig, TestStep @@ -27,7 +27,7 @@ class TestCaseRunner(object): # parse request_dict = step.request.dict() - parsed_request_dict = parse_content(request_dict, step.variables, self.config.functions) + parsed_request_dict = parse_data(request_dict, step.variables, self.config.functions) # prepare arguments method = parsed_request_dict.pop("method") From faa920d339d5d624143779fb0e7441695721db6a Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 11:59:55 +0800 Subject: [PATCH 14/49] fix parser: parse function args and kwargs before eval function --- httprunner/v3/parser.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index 8292bb36..1cfe8e98 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -234,7 +234,10 @@ def parse_string( function_meta = parse_function_params(func_params_str) args = function_meta["args"] kwargs = function_meta["kwargs"] - func_eval_value = func(*args, **kwargs) + + parsed_args = parse_data(args, variables_mapping, functions_mapping) + parsed_kwargs = parse_data(kwargs, variables_mapping, functions_mapping) + func_eval_value = func(*parsed_args, **parsed_kwargs) func_raw_str = "${" + func_name + f"({func_params_str})" + "}" if func_raw_str == raw_string: From 836e5fd64f993a9654a3600947827ec34b19a319 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 12:08:21 +0800 Subject: [PATCH 15/49] test: add unittests for parser --- httprunner/v3/parser_test.py | 236 +++++++++++++++++++++++++++++------ 1 file changed, 200 insertions(+), 36 deletions(-) diff --git a/httprunner/v3/parser_test.py b/httprunner/v3/parser_test.py index c63cab93..e7957614 100644 --- a/httprunner/v3/parser_test.py +++ b/httprunner/v3/parser_test.py @@ -159,40 +159,7 @@ class TestParserBasic(unittest.TestCase): [('func', '1, 2, a=3, b=4')] ) - def test_parse_content(self): - content = { - 'request': { - 'url': '/api/users/$uid', - 'method': "$method", - 'headers': {'token': '$token'}, - 'data': { - "null": None, - "true": True, - "false": False, - "empty_str": "", - "value": "abc${add_one(3)}def" - } - } - } - variables_mapping = { - "uid": 1000, - "method": "POST", - "token": "abc123" - } - functions_mapping = { - "add_one": lambda x: x + 1 - } - result = parser.parse_data(content, variables_mapping, functions_mapping) - self.assertEqual("/api/users/1000", result["request"]["url"]) - self.assertEqual("abc123", result["request"]["headers"]["token"]) - self.assertEqual("POST", result["request"]["method"]) - self.assertIsNone(result["request"]["data"]["null"]) - self.assertTrue(result["request"]["data"]["true"]) - self.assertFalse(result["request"]["data"]["false"]) - self.assertEqual("", result["request"]["data"]["empty_str"]) - self.assertEqual("abc4def", result["request"]["data"]["value"]) - - def test_parse_content_with_variables(self): + def test_parse_data_string_with_variables(self): variables_mapping = { "var_1": "abc", "var_2": "def", @@ -262,6 +229,56 @@ class TestParserBasic(unittest.TestCase): {"abc": "def"} ) + # format: $var + value = parser.parse_data("ABC$var_1", variables_mapping) + self.assertEqual(value, "ABCabc") + + value = parser.parse_data("ABC$var_1$var_3", variables_mapping) + self.assertEqual(value, "ABCabc123") + + value = parser.parse_data("ABC$var_1/$var_3", variables_mapping) + self.assertEqual(value, "ABCabc/123") + + value = parser.parse_data("ABC$var_1/", variables_mapping) + self.assertEqual(value, "ABCabc/") + + value = parser.parse_data("ABC$var_1$", variables_mapping) + self.assertEqual(value, "ABCabc$") + + value = parser.parse_data("ABC$var_1/123$var_1/456", variables_mapping) + self.assertEqual(value, "ABCabc/123abc/456") + + value = parser.parse_data("ABC$var_1/$var_2/$var_1", variables_mapping) + self.assertEqual(value, "ABCabc/def/abc") + + value = parser.parse_data("func1($var_1, $var_3)", variables_mapping) + self.assertEqual(value, "func1(abc, 123)") + + # format: ${var} + value = parser.parse_data("ABC${var_1}", variables_mapping) + self.assertEqual(value, "ABCabc") + + value = parser.parse_data("ABC${var_1}${var_3}", variables_mapping) + self.assertEqual(value, "ABCabc123") + + value = parser.parse_data("ABC${var_1}/${var_3}", variables_mapping) + self.assertEqual(value, "ABCabc/123") + + value = parser.parse_data("ABC${var_1}/", variables_mapping) + self.assertEqual(value, "ABCabc/") + + value = parser.parse_data("ABC${var_1}123", variables_mapping) + self.assertEqual(value, "ABCabc123") + + value = parser.parse_data("ABC${var_1}/123${var_1}/456", variables_mapping) + self.assertEqual(value, "ABCabc/123abc/456") + + value = parser.parse_data("ABC${var_1}/${var_2}/${var_1}", variables_mapping) + self.assertEqual(value, "ABCabc/def/abc") + + value = parser.parse_data("func1(${var_1}, ${var_3})", variables_mapping) + self.assertEqual(value, "func1(abc, 123)") + def test_parse_data_multiple_identical_variables(self): variables_mapping = { "var_1": "abc", @@ -293,11 +310,11 @@ class TestParserBasic(unittest.TestCase): "/users/100/1000/1498?userId=1000&data=1498" ) - def test_parse_data_functions(self): + def test_parse_data_string_with_functions(self): import random, string functions_mapping = { "gen_random_string": lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) \ - for _ in range(str_len)) + for _ in range(str_len)) } result = parser.parse_data("${gen_random_string(5)}", functions_mapping=functions_mapping) self.assertEqual(len(result), 5) @@ -320,6 +337,153 @@ class TestParserBasic(unittest.TestCase): with self.assertRaises(FunctionNotFound): parser.parse_data("/api/${gen_md5(abc)}") + variables_mapping = { + "var_1": "abc", + "var_2": "def", + "var_3": 123, + "var_4": {"a": 1}, + "var_5": True, + "var_6": None + } + functions_mapping = { + "func1": lambda x, y: str(x) + str(y) + } + + value = parser.parse_data("${func1($var_1, $var_3)}", variables_mapping, functions_mapping) + self.assertEqual(value, "abc123") + + value = parser.parse_data("ABC${func1($var_1, $var_3)}DE", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc123DE") + + value = parser.parse_data("ABC${func1($var_1, $var_3)}$var_5", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc123True") + + value = parser.parse_data("ABC${func1($var_1, $var_3)}DE$var_4", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc123DE{'a': 1}") + + value = parser.parse_data("ABC$var_5${func1($var_1, $var_3)}", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCTrueabc123") + + # TODO: Python builtin functions + # value = parser.parse_data("ABC${ord(a)}DEF${len(abcd)}", variables_mapping, functions_mapping) + # self.assertEqual(value, "ABC97DEF4") + + def test_parse_data_func_var_duplicate(self): + variables_mapping = { + "var_1": "abc", + "var_2": "def", + "var_3": 123, + "var_4": {"a": 1}, + "var_5": True, + "var_6": None + } + functions_mapping = { + "func1": lambda x, y: str(x) + str(y) + } + value = parser.parse_data( + "ABC${func1($var_1, $var_3)}--${func1($var_1, $var_3)}", + variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc123--abc123") + + value = parser.parse_data("ABC${func1($var_1, $var_3)}$var_1", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc123abc") + + value = parser.parse_data( + "ABC${func1($var_1, $var_3)}$var_1--${func1($var_1, $var_3)}$var_1", + variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc123abc--abc123abc") + + def test_parse_data_func_abnormal(self): + variables_mapping = { + "var_1": "abc", + "var_2": "def", + "var_3": 123, + "var_4": {"a": 1}, + "var_5": True, + "var_6": None + } + functions_mapping = { + "func1": lambda x, y: str(x) + str(y) + } + + # { + value = parser.parse_data("ABC$var_1{", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc{") + + value = parser.parse_data("{ABC$var_1{}a}", variables_mapping, functions_mapping) + self.assertEqual(value, "{ABCabc{}a}") + + value = parser.parse_data("AB{C$var_1{}a}", variables_mapping, functions_mapping) + self.assertEqual(value, "AB{Cabc{}a}") + + # } + value = parser.parse_data("ABC$var_1}", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc}") + + # $$ + value = parser.parse_data("ABC$$var_1{", variables_mapping, functions_mapping) + self.assertEqual(value, "ABC$var_1{") + + # $$$ + value = parser.parse_data("ABC$$$var_1{", variables_mapping, functions_mapping) + self.assertEqual(value, "ABC$abc{") + + # $$$$ + value = parser.parse_data("ABC$$$$var_1{", variables_mapping, functions_mapping) + self.assertEqual(value, "ABC$$var_1{") + + # ${ + value = parser.parse_data("ABC$var_1${", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc${") + + value = parser.parse_data("ABC$var_1${a", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc${a") + + # $} + value = parser.parse_data("ABC$var_1$}a", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc$}a") + + # }{ + value = parser.parse_data("ABC$var_1}{a", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc}{a") + + # {} + value = parser.parse_data("ABC$var_1{}a", variables_mapping, functions_mapping) + self.assertEqual(value, "ABCabc{}a") + + def test_parse_data_request(self): + content = { + 'request': { + 'url': '/api/users/$uid', + 'method': "$method", + 'headers': {'token': '$token'}, + 'data': { + "null": None, + "true": True, + "false": False, + "empty_str": "", + "value": "abc${add_one(3)}def" + } + } + } + variables_mapping = { + "uid": 1000, + "method": "POST", + "token": "abc123" + } + functions_mapping = { + "add_one": lambda x: x + 1 + } + result = parser.parse_data(content, variables_mapping, functions_mapping) + self.assertEqual("/api/users/1000", result["request"]["url"]) + self.assertEqual("abc123", result["request"]["headers"]["token"]) + self.assertEqual("POST", result["request"]["method"]) + self.assertIsNone(result["request"]["data"]["null"]) + self.assertTrue(result["request"]["data"]["true"]) + self.assertFalse(result["request"]["data"]["false"]) + self.assertEqual("", result["request"]["data"]["empty_str"]) + self.assertEqual("abc4def", result["request"]["data"]["value"]) + def test_parse_data_testcase(self): variables = { "uid": "1000", From 01e0b278b36a88883739a58fc5f6bf2d410e4fbf Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 12:54:57 +0800 Subject: [PATCH 16/49] feat: call with python builtin functions --- httprunner/v3/parser.py | 52 +++++++++++++++++++++++++++++++++--- httprunner/v3/parser_test.py | 5 ++-- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index 1cfe8e98..617b0b02 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -1,7 +1,9 @@ import ast +import builtins import re from typing import Any, Set, Text, Callable, Tuple, List, Dict, Union +from httprunner import loader, utils from httprunner.v3 import exceptions from httprunner.v3.exceptions import VariableNotFound, FunctionNotFound @@ -180,6 +182,51 @@ def parse_function_params(params): return function_meta +def get_mapping_function(function_name: Text, functions_mapping: Dict[Text, Callable]) -> Callable: + """ get function from functions_mapping, + if not found, then try to check if builtin function. + + Args: + function_name (str): function name + functions_mapping (dict): functions mapping + + Returns: + mapping function object. + + Raises: + exceptions.FunctionNotFound: function is neither defined in debugtalk.py nor builtin. + + """ + if function_name in functions_mapping: + return functions_mapping[function_name] + + elif function_name in ["parameterize", "P"]: + return loader.load_csv_file + + elif function_name in ["environ", "ENV"]: + return utils.get_os_environ + + elif function_name in ["multipart_encoder", "multipart_content_type"]: + # extension for upload test + from httprunner.ext import uploader + return getattr(uploader, function_name) + + try: + # check if HttpRunner builtin functions + built_in_functions = loader.load_builtin_functions() + return built_in_functions[function_name] + except KeyError: + pass + + try: + # check if Python builtin functions + return getattr(builtins, function_name) + except AttributeError: + pass + + raise exceptions.FunctionNotFound(f"{function_name} is not found.") + + def parse_string( raw_string: Text, variables_mapping: Dict[Text, Any], @@ -225,10 +272,7 @@ def parse_string( func_match = function_regex_compile.match(raw_string, match_start_position) if func_match: func_name = func_match.group(1) - try: - func = functions_mapping[func_name] - except KeyError: - raise FunctionNotFound(f"{func_name} not found in {functions_mapping}") + func = get_mapping_function(func_name, functions_mapping) func_params_str = func_match.group(2) function_meta = parse_function_params(func_params_str) diff --git a/httprunner/v3/parser_test.py b/httprunner/v3/parser_test.py index e7957614..ee42c525 100644 --- a/httprunner/v3/parser_test.py +++ b/httprunner/v3/parser_test.py @@ -364,9 +364,8 @@ class TestParserBasic(unittest.TestCase): value = parser.parse_data("ABC$var_5${func1($var_1, $var_3)}", variables_mapping, functions_mapping) self.assertEqual(value, "ABCTrueabc123") - # TODO: Python builtin functions - # value = parser.parse_data("ABC${ord(a)}DEF${len(abcd)}", variables_mapping, functions_mapping) - # self.assertEqual(value, "ABC97DEF4") + value = parser.parse_data("ABC${ord(a)}DEF${len(abcd)}", variables_mapping, functions_mapping) + self.assertEqual(value, "ABC97DEF4") def test_parse_data_func_var_duplicate(self): variables_mapping = { From 5dc580e93fe4371096c98405b8e745e989fdc99a Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 13:02:14 +0800 Subject: [PATCH 17/49] refactor: replace with get_mapping_variable --- httprunner/v3/parser.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index 617b0b02..c80c2a2e 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -182,6 +182,27 @@ def parse_function_params(params): return function_meta +def get_mapping_variable(variable_name: Text, variables_mapping: Dict[Text, Any]) -> Any: + """ get variable from variables_mapping. + + Args: + variable_name (str): variable name + variables_mapping (dict): variables mapping + + Returns: + mapping variable value. + + Raises: + exceptions.VariableNotFound: variable is not found. + + """ + # TODO: get variable from debugtalk module and environ + try: + return variables_mapping[variable_name] + except KeyError: + raise exceptions.VariableNotFound(f"{variable_name} not found in {variables_mapping}") + + def get_mapping_function(function_name: Text, functions_mapping: Dict[Text, Callable]) -> Callable: """ get function from functions_mapping, if not found, then try to check if builtin function. @@ -230,7 +251,7 @@ def get_mapping_function(function_name: Text, functions_mapping: Dict[Text, Call def parse_string( raw_string: Text, variables_mapping: Dict[Text, Any], - functions_mapping: Dict[Text, Callable]) -> Text: + functions_mapping: Dict[Text, Callable]) -> Any: """ parse string content with variables and functions mapping. Args: @@ -297,11 +318,7 @@ def parse_string( var_match = variable_regex_compile.match(raw_string, match_start_position) if var_match: var_name = var_match.group(1) or var_match.group(2) - # check if any variable undefined in variables_mapping - try: - var_value = variables_mapping[var_name] - except KeyError: - raise VariableNotFound(f"{var_name} not found in {variables_mapping}") + var_value = get_mapping_variable(var_name, variables_mapping) if f"${var_name}" == raw_string or "${" + var_name + "}" == raw_string: # raw_string is a variable, $var or ${var}, return its value directly From 58cb3b069162f661fe47f24d2496a51257439437 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 13:09:03 +0800 Subject: [PATCH 18/49] refactor: add argument typing --- httprunner/v3/parser.py | 28 ++++++++++++++-------------- httprunner/v3/runner.py | 6 +++--- httprunner/v3/schema/__init__.py | 9 +++++---- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index c80c2a2e..c0e12c7a 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -1,11 +1,11 @@ import ast import builtins import re -from typing import Any, Set, Text, Callable, Tuple, List, Dict, Union +from typing import Any, Set, Text, Callable, List, Dict from httprunner import loader, utils from httprunner.v3 import exceptions -from httprunner.v3.exceptions import VariableNotFound, FunctionNotFound +from httprunner.v3.schema import VariablesMapping, FunctionsMapping absolute_http_url_regexp = re.compile(r"^https?://", re.I) @@ -130,7 +130,7 @@ def extract_variables(content: Any) -> Set: return set() -def parse_function_params(params): +def parse_function_params(params: Text) -> Dict: """ parse function params to args and kwargs. Args: @@ -182,7 +182,7 @@ def parse_function_params(params): return function_meta -def get_mapping_variable(variable_name: Text, variables_mapping: Dict[Text, Any]) -> Any: +def get_mapping_variable(variable_name: Text, variables_mapping: VariablesMapping) -> Any: """ get variable from variables_mapping. Args: @@ -203,7 +203,7 @@ def get_mapping_variable(variable_name: Text, variables_mapping: Dict[Text, Any] raise exceptions.VariableNotFound(f"{variable_name} not found in {variables_mapping}") -def get_mapping_function(function_name: Text, functions_mapping: Dict[Text, Callable]) -> Callable: +def get_mapping_function(function_name: Text, functions_mapping: FunctionsMapping) -> Callable: """ get function from functions_mapping, if not found, then try to check if builtin function. @@ -250,8 +250,8 @@ def get_mapping_function(function_name: Text, functions_mapping: Dict[Text, Call def parse_string( raw_string: Text, - variables_mapping: Dict[Text, Any], - functions_mapping: Dict[Text, Callable]) -> Any: + variables_mapping: VariablesMapping, + functions_mapping: FunctionsMapping) -> Any: """ parse string content with variables and functions mapping. Args: @@ -346,8 +346,8 @@ def parse_string( def parse_data( raw_data: Any, - variables_mapping: Dict[Text, Any] = None, - functions_mapping: Dict[Text, Callable] = None) -> Any: + variables_mapping: VariablesMapping = None, + functions_mapping: FunctionsMapping = None) -> Any: """ parse raw data with evaluated variables mapping. Notice: variables_mapping should not contain any variable or function. """ @@ -379,10 +379,10 @@ def parse_data( def parse_variables_mapping( - variables_mapping: Dict[Text, Any], - functions_mapping: Dict[Text, Callable] = None) -> Dict[Text, Any]: + variables_mapping: VariablesMapping, + functions_mapping: FunctionsMapping = None) -> VariablesMapping: - parsed_variables: Dict[Text, Any] = {} + parsed_variables: VariablesMapping = {} while len(parsed_variables) != len(variables_mapping): for var_name in variables_mapping: @@ -409,12 +409,12 @@ def parse_variables_mapping( if not_defined_variables: # e.g. {"varA": "123$varB", "varB": "456$varC"} # e.g. {"varC": "${sum_two($a, $b)}"} - raise VariableNotFound(not_defined_variables) + raise exceptions.VariableNotFound(not_defined_variables) try: parsed_value = parse_data( var_value, parsed_variables, functions_mapping) - except VariableNotFound: + except exceptions.VariableNotFound: continue parsed_variables[var_name] = parsed_value diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 3e3c3489..8ab7113b 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -5,7 +5,7 @@ from loguru import logger from httprunner.v3.parser import build_url, parse_data, parse_variables_mapping from httprunner.v3.response import ResponseObject -from httprunner.v3.schema import TestsConfig, TestStep +from httprunner.v3.schema import TestsConfig, TestStep, VariablesMapping class TestCaseRunner(object): @@ -18,11 +18,11 @@ class TestCaseRunner(object): self.session = s return self - def with_variables(self, **variables) -> "TestCaseRunner": + def with_variables(self, **variables: VariablesMapping) -> "TestCaseRunner": self.config.variables.update(variables) return self - def run_step(self, step): + def run_step(self, step: TestStep): logger.info(f"run step: {step.name}") # parse diff --git a/httprunner/v3/schema/__init__.py b/httprunner/v3/schema/__init__.py index ea1220d2..198e4b57 100644 --- a/httprunner/v3/schema/__init__.py +++ b/httprunner/v3/schema/__init__.py @@ -8,7 +8,8 @@ from pydantic import HttpUrl Name = Text Url = Text BaseUrl = Union[HttpUrl, Text] -Variables = Dict[Text, Any] +VariablesMapping = Dict[Text, Any] +FunctionsMapping = Dict[Text, Callable] Headers = Dict[Text, Text] Verify = bool Hook = List[Text] @@ -33,8 +34,8 @@ class TestsConfig(BaseModel): name: Name verify: Verify = False base_url: BaseUrl = "" - variables: Variables = {} - functions: Dict[Text, Callable] + variables: VariablesMapping = {} + functions: FunctionsMapping = {} setup_hooks: Hook = [] teardown_hooks: Hook = [] export: Export = [] @@ -56,6 +57,6 @@ class Request(BaseModel): class TestStep(BaseModel): name: Name request: Request - variables: Variables = {} + variables: VariablesMapping = {} extract: Dict[Text, Text] = {} validation: Validate = Field([], alias="validate") From 6a5e271acd64fc442995a3ff9af4e148fb67b6a6 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 13:30:36 +0800 Subject: [PATCH 19/49] refactor: change example file name --- .../{with_functions.yml => request_with_functions.yml} | 0 .../{with_functions_test.py => request_with_functions_test.py} | 0 .../{with_variables.yml => request_with_variables.yml} | 0 .../{with_variables_test.py => request_with_variables_test.py} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename examples/postman_echo/request_methods/{with_functions.yml => request_with_functions.yml} (100%) rename examples/postman_echo/request_methods/{with_functions_test.py => request_with_functions_test.py} (100%) rename examples/postman_echo/request_methods/{with_variables.yml => request_with_variables.yml} (100%) rename examples/postman_echo/request_methods/{with_variables_test.py => request_with_variables_test.py} (100%) diff --git a/examples/postman_echo/request_methods/with_functions.yml b/examples/postman_echo/request_methods/request_with_functions.yml similarity index 100% rename from examples/postman_echo/request_methods/with_functions.yml rename to examples/postman_echo/request_methods/request_with_functions.yml diff --git a/examples/postman_echo/request_methods/with_functions_test.py b/examples/postman_echo/request_methods/request_with_functions_test.py similarity index 100% rename from examples/postman_echo/request_methods/with_functions_test.py rename to examples/postman_echo/request_methods/request_with_functions_test.py diff --git a/examples/postman_echo/request_methods/with_variables.yml b/examples/postman_echo/request_methods/request_with_variables.yml similarity index 100% rename from examples/postman_echo/request_methods/with_variables.yml rename to examples/postman_echo/request_methods/request_with_variables.yml diff --git a/examples/postman_echo/request_methods/with_variables_test.py b/examples/postman_echo/request_methods/request_with_variables_test.py similarity index 100% rename from examples/postman_echo/request_methods/with_variables_test.py rename to examples/postman_echo/request_methods/request_with_variables_test.py From 0532e0d35dfa7806e9d1066e45d227709581973c Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 14:26:57 +0800 Subject: [PATCH 20/49] feat: validate with config/teststep/extracted variables --- .../validate_with_variables.yml | 58 +++++++++++ .../validate_with_variables_test.py | 98 +++++++++++++++++++ httprunner/v3/response.py | 9 +- httprunner/v3/runner.py | 12 ++- httprunner/v3/schema/__init__.py | 4 +- 5 files changed, 173 insertions(+), 8 deletions(-) create mode 100644 examples/postman_echo/request_methods/validate_with_variables.yml create mode 100644 examples/postman_echo/request_methods/validate_with_variables_test.py diff --git a/examples/postman_echo/request_methods/validate_with_variables.yml b/examples/postman_echo/request_methods/validate_with_variables.yml new file mode 100644 index 00000000..7f77219f --- /dev/null +++ b/examples/postman_echo/request_methods/validate_with_variables.yml @@ -0,0 +1,58 @@ +config: + name: "request methods testcase: validate with variables" + variables: + foo1: session_bar1 + base_url: "https://postman-echo.com" + verify: False + +teststeps: +- + name: get with params + variables: + foo1: bar1 + foo2: session_bar2 + request: + method: GET + url: /get + params: + foo1: $foo1 + foo2: $foo2 + headers: + User-Agent: HttpRunner/3.0 + extract: + session_foo2: "body.args.foo2" + validate: + - eq: ["status_code", 200] + - eq: ["body.args.foo1", "$foo1"] + - eq: ["body.args.foo2", "$foo2"] +- + name: post raw text + variables: + foo1: "hello world" + foo3: "$session_foo2" + request: + method: POST + url: /post + headers: + User-Agent: HttpRunner/3.0 + Content-Type: "text/plain" + data: "This is expected to be sent back as part of response body: $foo1-$foo3." + validate: + - eq: ["status_code", 200] + - eq: ["body.data", "This is expected to be sent back as part of response body: session_bar1-$foo3."] +- + name: post form data + variables: + foo1: bar1 + foo2: bar2 + request: + method: POST + url: /post + headers: + User-Agent: HttpRunner/3.0 + Content-Type: "application/x-www-form-urlencoded" + data: "foo1=$foo1&foo2=$foo2" + validate: + - eq: ["status_code", 200] + - eq: ["body.form.foo1", "$foo1"] + - eq: ["body.form.foo2", "$foo2"] diff --git a/examples/postman_echo/request_methods/validate_with_variables_test.py b/examples/postman_echo/request_methods/validate_with_variables_test.py new file mode 100644 index 00000000..14cd2e71 --- /dev/null +++ b/examples/postman_echo/request_methods/validate_with_variables_test.py @@ -0,0 +1,98 @@ +from httprunner.v3.runner import TestCaseRunner +from httprunner.v3.schema import TestsConfig, TestStep + + +class TestCaseRequestMethodsValidateWithVariables(TestCaseRunner): + config = TestsConfig(**{ + "name": "request methods testcase: validate with variables", + "variables": { + "foo1": "session_bar1" + }, + "base_url": "https://postman-echo.com", + "verify": False + }) + + teststeps = [ + TestStep(**{ + "name": "get with params", + "variables": { + "foo1": "bar1", + "foo2": "session_bar2" + }, + "request": { + "method": "GET", + "url": "/get", + "params": { + "foo1": "$foo1", + "foo2": "$foo2" + }, + "headers": { + "User-Agent": "HttpRunner/3.0" + } + }, + "extract": { + "session_foo2": "body.args.foo2" + }, + "validate": [ + {"eq": ["status_code", 200]}, + {"eq": ["body.args.foo1", "session_bar1"]}, + {"eq": ["body.args.foo1", "$foo1"]}, + {"eq": ["body.args.foo2", "session_bar2"]}, + {"eq": ["body.args.foo2", "$foo2"]} + ] + }), + TestStep(**{ + "name": "post raw text", + "variables": { + "foo1": "hello world", + "foo3": "$session_foo2" + }, + "request": { + "method": "POST", + "url": "/post", + "data": "This is expected to be sent back as part of response body: $foo1-$foo3.", + "headers": { + "User-Agent": "HttpRunner/3.0", + "Content-Type": "text/plain" + } + }, + "validate": [ + {"eq": ["status_code", 200]}, + {"eq": [ + "body.data", + "This is expected to be sent back as part of response body: session_bar1-session_bar2." + ]}, + {"eq": [ + "body.data", + "This is expected to be sent back as part of response body: $foo1-$foo3." + ]}, + ] + }), + TestStep(**{ + "name": "post form data", + "variables": { + "foo1": "session_bar1", + "foo2": "bar2" + }, + "request": { + "method": "POST", + "url": "/post", + "data": "foo1=$foo1&foo2=$foo2", + "headers": { + "User-Agent": "HttpRunner/3.0", + "Content-Type": "application/x-www-form-urlencoded" + } + }, + "validate": [ + {"eq": ["status_code", 200]}, + {"eq": ["body.form.foo1", "session_bar1"]}, + {"eq": ["body.form.foo1", "$foo1"]}, + {"eq": ["body.form.foo2", "bar2"]}, + {"eq": ["body.form.foo2", "$foo2"]} + ] + }) + ] + + +if __name__ == '__main__': + TestCaseRequestMethodsValidateWithVariables().run() diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py index 5dbf9577..222e63b7 100644 --- a/httprunner/v3/response.py +++ b/httprunner/v3/response.py @@ -1,10 +1,12 @@ -from typing import Dict, Text, Any +from typing import Dict, Text, Any, NoReturn import jmespath import requests from loguru import logger from httprunner.v3.exceptions import ParamsError, ValidationFailure +from httprunner.v3.parser import parse_data +from httprunner.v3.schema import VariablesMapping, Validators from httprunner.v3.validator import uniform_validator, AssertMethods @@ -23,7 +25,7 @@ class ResponseObject(object): "body": resp_obj.json() } - def validate(self, validators): + def validate(self, validators: Validators, variables_mapping: VariablesMapping = None) -> NoReturn: for v in validators: u_validator = uniform_validator(v) @@ -39,6 +41,9 @@ class ResponseObject(object): except AttributeError: raise ParamsError(f"Assert Method not supported: {assert_method}") + # parse expected value with config/teststep/extracted variables + expect_value = parse_data(expect_value, variables_mapping) + try: assert_func(actual_value, expect_value) msg += " - success" diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 8ab7113b..45c92ad3 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -44,13 +44,17 @@ class TestCaseRunner(object): resp = session.request(method, url, **parsed_request_dict) resp_obj = ResponseObject(resp) - # validate - validators = step.validation - resp_obj.validate(validators) - # extract extractors = step.extract extract_mapping = resp_obj.extract(extractors) + + variables_mapping = step.variables + variables_mapping.update(extract_mapping) + + # validate + validators = step.validation + resp_obj.validate(validators, variables_mapping) + return extract_mapping def test_start(self): diff --git a/httprunner/v3/schema/__init__.py b/httprunner/v3/schema/__init__.py index 198e4b57..00577a4d 100644 --- a/httprunner/v3/schema/__init__.py +++ b/httprunner/v3/schema/__init__.py @@ -14,7 +14,7 @@ Headers = Dict[Text, Text] Verify = bool Hook = List[Text] Export = List[Text] -Validate = List[Dict] +Validators = List[Dict] Env = Dict[Text, Any] @@ -59,4 +59,4 @@ class TestStep(BaseModel): request: Request variables: VariablesMapping = {} extract: Dict[Text, Text] = {} - validation: Validate = Field([], alias="validate") + validation: Validators = Field([], alias="validate") From 96b5a127cd6433559e77c74657ef78c18cd5eef2 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 14:33:01 +0800 Subject: [PATCH 21/49] refactor: rename TestStep model field, validators --- httprunner/v3/runner.py | 2 +- httprunner/v3/schema/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 45c92ad3..0048de7e 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -52,7 +52,7 @@ class TestCaseRunner(object): variables_mapping.update(extract_mapping) # validate - validators = step.validation + validators = step.validators resp_obj.validate(validators, variables_mapping) return extract_mapping diff --git a/httprunner/v3/schema/__init__.py b/httprunner/v3/schema/__init__.py index 00577a4d..ae426451 100644 --- a/httprunner/v3/schema/__init__.py +++ b/httprunner/v3/schema/__init__.py @@ -59,4 +59,4 @@ class TestStep(BaseModel): request: Request variables: VariablesMapping = {} extract: Dict[Text, Text] = {} - validation: Validators = Field([], alias="validate") + validators: Validators = Field([], alias="validate") From 41f819b9cb359062798e17036e50f9f2d3feb018 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 14:45:24 +0800 Subject: [PATCH 22/49] feat: validate with functions --- .../validate_with_functions.yml | 60 +++++++++++++++++++ .../validate_with_functions_test.py | 53 ++++++++++++++++ httprunner/v3/response.py | 12 ++-- httprunner/v3/runner.py | 2 +- 4 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 examples/postman_echo/request_methods/validate_with_functions.yml create mode 100644 examples/postman_echo/request_methods/validate_with_functions_test.py diff --git a/examples/postman_echo/request_methods/validate_with_functions.yml b/examples/postman_echo/request_methods/validate_with_functions.yml new file mode 100644 index 00000000..ea2c6a40 --- /dev/null +++ b/examples/postman_echo/request_methods/validate_with_functions.yml @@ -0,0 +1,60 @@ +config: + name: "request methods testcase: validate with functions" + variables: + foo1: session_bar1 + base_url: "https://postman-echo.com" + verify: False + +teststeps: +- + name: get with params + variables: + foo1: bar1 + foo2: session_bar2 + sum_v: "${sum_two(1, 2)}" + request: + method: GET + url: /get + params: + foo1: $foo1 + foo2: $foo2 + sum_v: $sum_v + headers: + User-Agent: HttpRunner/${get_httprunner_version()} + extract: + session_foo2: "body.args.foo2" + validate: + - eq: ["status_code", 200] + - eq: ["body.args.sum_v", "3"] + - less_than: ["body.args.sum_v", "${sum_two(2, 2)}"] +- + name: post raw text + variables: + foo1: "hello world" + foo3: "$session_foo2" + request: + method: POST + url: /post + headers: + User-Agent: HttpRunner/${get_httprunner_version()} + Content-Type: "text/plain" + data: "This is expected to be sent back as part of response body: $foo1-$foo3." + validate: + - eq: ["status_code", 200] + - eq: ["body.data", "This is expected to be sent back as part of response body: session_bar1-session_bar2."] +- + name: post form data + variables: + foo1: bar1 + foo2: bar2 + request: + method: POST + url: /post + headers: + User-Agent: HttpRunner/${get_httprunner_version()} + Content-Type: "application/x-www-form-urlencoded" + data: "foo1=$foo1&foo2=$foo2" + validate: + - eq: ["status_code", 200] + - eq: ["body.form.foo1", "session_bar1"] + - eq: ["body.form.foo2", "bar2"] diff --git a/examples/postman_echo/request_methods/validate_with_functions_test.py b/examples/postman_echo/request_methods/validate_with_functions_test.py new file mode 100644 index 00000000..8f600dfb --- /dev/null +++ b/examples/postman_echo/request_methods/validate_with_functions_test.py @@ -0,0 +1,53 @@ +from httprunner.v3.runner import TestCaseRunner +from httprunner.v3.schema import TestsConfig, TestStep +from examples.postman_echo import debugtalk + + +class TestCaseRequestMethodsValidateWithFunctions(TestCaseRunner): + config = TestsConfig(**{ + "name": "request methods testcase: validate with functions", + "variables": { + "foo1": "session_bar1" + }, + "functions": { + "get_httprunner_version": debugtalk.get_httprunner_version, + "sum_two": debugtalk.sum_two + }, + "base_url": "https://postman-echo.com", + "verify": False + }) + + teststeps = [ + TestStep(**{ + "name": "get with params", + "variables": { + "foo1": "bar1", + "foo2": "session_bar2", + "sum_v": "${sum_two(1, 2)}" + }, + "request": { + "method": "GET", + "url": "/get", + "params": { + "foo1": "$foo1", + "foo2": "$foo2", + "sum_v": "$sum_v" + }, + "headers": { + "User-Agent": "HttpRunner/${get_httprunner_version()}" + } + }, + "extract": { + "session_foo2": "body.args.foo2" + }, + "validate": [ + {"eq": ["status_code", 200]}, + {"eq": ["body.args.sum_v", 3]}, + {"less_than": ["body.args.sum_v", "${sum_two(2, 2)}"]} + ] + }) + ] + + +if __name__ == '__main__': + TestCaseRequestMethodsValidateWithFunctions().run() diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py index 222e63b7..8523a688 100644 --- a/httprunner/v3/response.py +++ b/httprunner/v3/response.py @@ -5,8 +5,8 @@ import requests from loguru import logger from httprunner.v3.exceptions import ParamsError, ValidationFailure -from httprunner.v3.parser import parse_data -from httprunner.v3.schema import VariablesMapping, Validators +from httprunner.v3.parser import parse_data, parse_string_value +from httprunner.v3.schema import VariablesMapping, Validators, FunctionsMapping from httprunner.v3.validator import uniform_validator, AssertMethods @@ -25,7 +25,10 @@ class ResponseObject(object): "body": resp_obj.json() } - def validate(self, validators: Validators, variables_mapping: VariablesMapping = None) -> NoReturn: + def validate(self, + validators: Validators, + variables_mapping: VariablesMapping = None, + functions_mapping: FunctionsMapping = None) -> NoReturn: for v in validators: u_validator = uniform_validator(v) @@ -41,8 +44,9 @@ class ResponseObject(object): except AttributeError: raise ParamsError(f"Assert Method not supported: {assert_method}") + actual_value = parse_string_value(actual_value) # parse expected value with config/teststep/extracted variables - expect_value = parse_data(expect_value, variables_mapping) + expect_value = parse_data(expect_value, variables_mapping, functions_mapping) try: assert_func(actual_value, expect_value) diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 0048de7e..53b27d11 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -53,7 +53,7 @@ class TestCaseRunner(object): # validate validators = step.validators - resp_obj.validate(validators, variables_mapping) + resp_obj.validate(validators, variables_mapping, self.config.functions) return extract_mapping From 94cf2b074fa6c0250e807e068b770ea7b2e095ef Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 14:54:28 +0800 Subject: [PATCH 23/49] refactor: validate with builtin assert methods --- httprunner/v3/response.py | 12 ++++-------- httprunner/v3/validator.py | 15 --------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py index 8523a688..9c35ae4d 100644 --- a/httprunner/v3/response.py +++ b/httprunner/v3/response.py @@ -4,10 +4,10 @@ import jmespath import requests from loguru import logger -from httprunner.v3.exceptions import ParamsError, ValidationFailure -from httprunner.v3.parser import parse_data, parse_string_value +from httprunner.v3.exceptions import ValidationFailure +from httprunner.v3.parser import parse_data, parse_string_value, get_mapping_function from httprunner.v3.schema import VariablesMapping, Validators, FunctionsMapping -from httprunner.v3.validator import uniform_validator, AssertMethods +from httprunner.v3.validator import uniform_validator class ResponseObject(object): @@ -39,11 +39,7 @@ class ResponseObject(object): msg = f"assert {field} {assert_method} {expect_value}" - try: - assert_func = getattr(AssertMethods, assert_method) - except AttributeError: - raise ParamsError(f"Assert Method not supported: {assert_method}") - + assert_func = get_mapping_function(assert_method, functions_mapping) actual_value = parse_string_value(actual_value) # parse expected value with config/teststep/extracted variables expect_value = parse_data(expect_value, variables_mapping, functions_mapping) diff --git a/httprunner/v3/validator.py b/httprunner/v3/validator.py index cc0b39b9..c1278c2e 100644 --- a/httprunner/v3/validator.py +++ b/httprunner/v3/validator.py @@ -89,18 +89,3 @@ def uniform_validator(validator): "expect": expect_value, "assert": assert_method } - - -class AssertMethods(object): - - @staticmethod - def equals(actual_value, expect_value): - assert actual_value == expect_value - - @staticmethod - def less_than(actual_value, expect_value): - assert actual_value < expect_value - - @staticmethod - def greater_than(actual_value, expect_value): - assert actual_value > expect_value From a4328b7586976005621b11ef08dcb383ee61b82f Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 15:13:48 +0800 Subject: [PATCH 24/49] fix: show assert value type when validation failed --- .../request_with_functions_test.py | 2 +- .../validate_with_functions.yml | 31 ------------------- httprunner/v3/response.py | 4 ++- 3 files changed, 4 insertions(+), 33 deletions(-) diff --git a/examples/postman_echo/request_methods/request_with_functions_test.py b/examples/postman_echo/request_methods/request_with_functions_test.py index b8bd9126..3de4c44a 100644 --- a/examples/postman_echo/request_methods/request_with_functions_test.py +++ b/examples/postman_echo/request_methods/request_with_functions_test.py @@ -44,7 +44,7 @@ class TestCaseRequestMethodsWithFunctions(TestCaseRunner): {"eq": ["status_code", 200]}, {"eq": ["body.args.foo1", "session_bar1"]}, {"eq": ["body.args.foo2", "session_bar2"]}, - {"eq": ["body.args.sum_v", "3"]} + {"eq": ["body.args.sum_v", 3]} ] }), TestStep(**{ diff --git a/examples/postman_echo/request_methods/validate_with_functions.yml b/examples/postman_echo/request_methods/validate_with_functions.yml index ea2c6a40..8de9242b 100644 --- a/examples/postman_echo/request_methods/validate_with_functions.yml +++ b/examples/postman_echo/request_methods/validate_with_functions.yml @@ -27,34 +27,3 @@ teststeps: - eq: ["status_code", 200] - eq: ["body.args.sum_v", "3"] - less_than: ["body.args.sum_v", "${sum_two(2, 2)}"] -- - name: post raw text - variables: - foo1: "hello world" - foo3: "$session_foo2" - request: - method: POST - url: /post - headers: - User-Agent: HttpRunner/${get_httprunner_version()} - Content-Type: "text/plain" - data: "This is expected to be sent back as part of response body: $foo1-$foo3." - validate: - - eq: ["status_code", 200] - - eq: ["body.data", "This is expected to be sent back as part of response body: session_bar1-session_bar2."] -- - name: post form data - variables: - foo1: bar1 - foo2: bar2 - request: - method: POST - url: /post - headers: - User-Agent: HttpRunner/${get_httprunner_version()} - Content-Type: "application/x-www-form-urlencoded" - data: "foo1=$foo1&foo2=$foo2" - validate: - - eq: ["status_code", 200] - - eq: ["body.form.foo1", "session_bar1"] - - eq: ["body.form.foo2", "bar2"] diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py index 9c35ae4d..d4546962 100644 --- a/httprunner/v3/response.py +++ b/httprunner/v3/response.py @@ -51,7 +51,9 @@ class ResponseObject(object): except AssertionError: msg += " - fail" logger.error(msg) - raise ValidationFailure(f"assert {field}: {actual_value} {assert_method} {expect_value}") + actual_type = type(actual_value).__name__ + expect_type = type(expect_value).__name__ + raise ValidationFailure(f"assert {field}: {actual_value}({actual_type}) {assert_method} {expect_value}({expect_type})") def extract(self, extractors: Dict[Text, Text]) -> Dict[Text, Any]: if not extractors: From a98d4113340029e49664f869c105f467e0c32709 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 18:55:31 +0800 Subject: [PATCH 25/49] feat: get request & response meta datas --- .../validate_with_functions.yml | 2 +- .../validate_with_variables_test.py | 3 ++- httprunner/v3/runner.py | 23 ++++++++++++++----- httprunner/v3/schema/__init__.py | 23 ++++++++++++++++++- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/examples/postman_echo/request_methods/validate_with_functions.yml b/examples/postman_echo/request_methods/validate_with_functions.yml index 8de9242b..41aca935 100644 --- a/examples/postman_echo/request_methods/validate_with_functions.yml +++ b/examples/postman_echo/request_methods/validate_with_functions.yml @@ -25,5 +25,5 @@ teststeps: session_foo2: "body.args.foo2" validate: - eq: ["status_code", 200] - - eq: ["body.args.sum_v", "3"] + - eq: ["body.args.sum_v", 3] - less_than: ["body.args.sum_v", "${sum_two(2, 2)}"] diff --git a/examples/postman_echo/request_methods/validate_with_variables_test.py b/examples/postman_echo/request_methods/validate_with_variables_test.py index 14cd2e71..840e5ce1 100644 --- a/examples/postman_echo/request_methods/validate_with_variables_test.py +++ b/examples/postman_echo/request_methods/validate_with_variables_test.py @@ -95,4 +95,5 @@ class TestCaseRequestMethodsValidateWithVariables(TestCaseRunner): if __name__ == '__main__': - TestCaseRequestMethodsValidateWithVariables().run() + runner = TestCaseRequestMethodsValidateWithVariables().run() + print(runner.meta_datas) diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 53b27d11..297aac63 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -1,20 +1,26 @@ from typing import List -import requests from loguru import logger +from httprunner.client import HttpSession from httprunner.v3.parser import build_url, parse_data, parse_variables_mapping from httprunner.v3.response import ResponseObject -from httprunner.v3.schema import TestsConfig, TestStep, VariablesMapping +from httprunner.v3.schema import TestsConfig, TestStep, VariablesMapping, TestCase class TestCaseRunner(object): config: TestsConfig = {} teststeps: List[TestStep] = [] - session: requests.Session = None + session: HttpSession = None + meta_datas: List = [] - def with_session(self, s: requests.Session) -> "TestCaseRunner": + def init(self, testcase: TestCase) -> "TestCaseRunner": + self.config = testcase.config + self.teststeps = testcase.teststeps + return self + + def with_session(self, s: HttpSession) -> "TestCaseRunner": self.session = s return self @@ -40,8 +46,8 @@ class TestCaseRunner(object): logger.debug(f"request kwargs(raw): {parsed_request_dict}") # request - session = self.session or requests.Session() - resp = session.request(method, url, **parsed_request_dict) + self.session = self.session or HttpSession() + resp = self.session.request(method, url, **parsed_request_dict) resp_obj = ResponseObject(resp) # extract @@ -59,6 +65,7 @@ class TestCaseRunner(object): def test_start(self): """main entrance""" + self.meta_datas.clear() session_variables = {} for step in self.teststeps: # update with config variables @@ -71,6 +78,10 @@ class TestCaseRunner(object): extract_mapping = self.run_step(step) # save extracted variables to session variables session_variables.update(extract_mapping) + # save request & response meta data + self.meta_datas.append(self.session.meta_data) + + return self def run(self): """main entrance alias for test_start""" diff --git a/httprunner/v3/schema/__init__.py b/httprunner/v3/schema/__init__.py index ae426451..5d75ac71 100644 --- a/httprunner/v3/schema/__init__.py +++ b/httprunner/v3/schema/__init__.py @@ -1,6 +1,7 @@ from enum import Enum from typing import Any -from typing import Dict, List, Text, Union, Callable +from typing import Dict, Text, Union, Callable +from typing import List from pydantic import BaseModel, Field from pydantic import HttpUrl @@ -56,7 +57,27 @@ class Request(BaseModel): class TestStep(BaseModel): name: Name + times: int = 1 request: Request variables: VariablesMapping = {} extract: Dict[Text, Text] = {} validators: Validators = Field([], alias="validate") + + +class TestCase(BaseModel): + config: TestsConfig + teststeps: List[TestStep] + + +class ProjectMeta(BaseModel): + debugtalk_py: Text = "" + variables: VariablesMapping = {} + functions: FunctionsMapping = {} + env: Env = {} + PWD: Text + test_path: Text + + +class TestsMapping(BaseModel): + project_mapping: ProjectMeta # TODO: rename to project_meta + testcases: List[TestCase] From f6a6e91c864126b140155ba3d302d62ddaee3dd1 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 21:19:05 +0800 Subject: [PATCH 26/49] fix: record teststep name --- httprunner/v3/runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 297aac63..cf94d148 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -79,6 +79,7 @@ class TestCaseRunner(object): # save extracted variables to session variables session_variables.update(extract_mapping) # save request & response meta data + self.session.meta_data["name"] = step.name self.meta_datas.append(self.session.meta_data) return self From 8620c2e1fafb80391ba8988d97977476494048ea Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 21:22:00 +0800 Subject: [PATCH 27/49] refactor: make run_step as private method --- httprunner/v3/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index cf94d148..bbf2a30f 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -28,7 +28,7 @@ class TestCaseRunner(object): self.config.variables.update(variables) return self - def run_step(self, step: TestStep): + def __run_step(self, step: TestStep): logger.info(f"run step: {step.name}") # parse @@ -75,7 +75,7 @@ class TestCaseRunner(object): # parse variables step.variables = parse_variables_mapping(step.variables, self.config.functions) # run step - extract_mapping = self.run_step(step) + extract_mapping = self.__run_step(step) # save extracted variables to session variables session_variables.update(extract_mapping) # save request & response meta data From 71e65e02d52cb82ff671e4e5461d00022f4db030 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 21:31:09 +0800 Subject: [PATCH 28/49] feat: implement HttpRunner main interface v3 --- httprunner/v3/api.py | 256 ++++++++++++++++++++++++++++++++++++++ httprunner/v3/api_test.py | 17 +++ 2 files changed, 273 insertions(+) create mode 100644 httprunner/v3/api.py create mode 100644 httprunner/v3/api_test.py diff --git a/httprunner/v3/api.py b/httprunner/v3/api.py new file mode 100644 index 00000000..e77f8a30 --- /dev/null +++ b/httprunner/v3/api.py @@ -0,0 +1,256 @@ +import os +import sys +import unittest +from typing import List, Tuple + +from loguru import logger + +from httprunner import report, loader, utils, exceptions, __version__ +from httprunner.v3.runner import TestCaseRunner +from httprunner.v3.schema import TestsMapping + + +class HttpRunner(object): + """ Developer Interface: Main Interface + Usage: + + from httprunner.api import HttpRunner + runner = HttpRunner( + failfast=True, + save_tests=True, + log_level="INFO", + log_file="test.log" + ) + summary = runner.run(path_or_tests) + + """ + + def __init__(self, failfast=False, save_tests=False, log_level="WARNING", log_file=None): + """ initialize HttpRunner. + + Args: + failfast (bool): stop the test run on the first error or failure. + save_tests (bool): save loaded/parsed tests to JSON file. + log_level (str): logging level. + log_file (str): log file path. + + """ + self.exception_stage = "initialize HttpRunner()" + kwargs = { + "failfast": failfast, + "resultclass": report.HtmlTestResult + } + + logger.remove() + log_level = log_level.upper() + logger.add(sys.stdout, level=log_level) + if log_file: + logger.add(log_file, level=log_level) + + self.unittest_runner = unittest.TextTestRunner(**kwargs) + self.test_loader = unittest.TestLoader() + self.save_tests = save_tests + self._summary = None + self.test_path = None + + def _prepare_tests(self, tests: TestsMapping) -> List[unittest.TestSuite]: + def _add_test(test_runner: TestCaseRunner): + """ add test to testcase. + """ + def test(self): + try: + test_runner.run() + except exceptions.MyBaseFailure as ex: + self.fail(str(ex)) + finally: + self.meta_datas = test_runner.meta_datas + + test.__doc__ = test_runner.config.name + return test + + project_meta = tests.project_mapping + testcases = tests.testcases + + prepared_testcases: List[unittest.TestSuite] = [] + + for testcase in testcases: + testcase.config.variables.update(project_meta.variables) + testcase.config.functions.update(project_meta.functions) + + test_runner = TestCaseRunner().init(testcase) + + TestSequense = type('TestSequense', (unittest.TestCase,), {}) + test_method = _add_test(test_runner) + setattr(TestSequense, "test_method_name", test_method) + + loaded_testcase = self.test_loader.loadTestsFromTestCase(TestSequense) + setattr(loaded_testcase, "config", testcase.config) + # setattr(loaded_testcase, "teststeps", testcase.teststeps) + # setattr(loaded_testcase, "runner", test_runner) + prepared_testcases.append(loaded_testcase) + + return prepared_testcases + + def _run_suite(self, prepared_testcases: List[unittest.TestSuite]) -> List[Tuple]: + """ run prepared testcases + """ + tests_results: List[Tuple] = [] + + for index, testcase in enumerate(prepared_testcases): + log_handler = None + if self.save_tests: + logs_file_abs_path = utils.prepare_log_file_abs_path( + self.test_path, f"testcase_{index+1}.log" + ) + log_handler = logger.add(logs_file_abs_path, level="DEBUG") + + logger.info(f"Start to run testcase: {testcase.config.name}") + + result = self.unittest_runner.run(testcase) + if result.wasSuccessful(): + tests_results.append((testcase, result)) + else: + tests_results.insert(0, (testcase, result)) + + if self.save_tests and log_handler: + logger.remove(log_handler) + + return tests_results + + def _aggregate(self, tests_results: List[Tuple]): + """ aggregate results + + Args: + tests_results (list): list of (testcase, result) + + """ + summary = { + "success": True, + "stat": { + "testcases": { + "total": len(tests_results), + "success": 0, + "fail": 0 + }, + "teststeps": {} + }, + "time": {}, + "platform": report.get_platform(), + "details": [] + } + + for index, tests_result in enumerate(tests_results): + testcase, result = tests_result + testcase_summary = report.get_summary(result) + + if testcase_summary["success"]: + summary["stat"]["testcases"]["success"] += 1 + else: + summary["stat"]["testcases"]["fail"] += 1 + + summary["success"] &= testcase_summary["success"] + testcase_summary["name"] = testcase.config.name + testcase_summary["in_out"] = { + "in": testcase.config.variables, + "out": testcase.config.export + } + + report.aggregate_stat(summary["stat"]["teststeps"], testcase_summary["stat"]) + report.aggregate_stat(summary["time"], testcase_summary["time"]) + + if self.save_tests: + logs_file_abs_path = utils.prepare_log_file_abs_path( + self.test_path, f"testcase_{index+1}.log" + ) + testcase_summary["log"] = logs_file_abs_path + + summary["details"].append(testcase_summary) + + return summary + + def run_tests(self, tests_mapping): + """ run testcase/testsuite data + """ + tests = TestsMapping.parse_obj(tests_mapping) + self.test_path = tests.project_mapping.test_path + + if self.save_tests: + utils.dump_json_file( + tests_mapping, + utils.prepare_log_file_abs_path(self.test_path, "loaded.json") + ) + + # prepare testcases + self.exception_stage = "prepare testcases" + prepared_testcases = self._prepare_tests(tests) + + # run prepared testcases + self.exception_stage = "run prepared testcases" + results = self._run_suite(prepared_testcases) + + # aggregate results + self.exception_stage = "aggregate results" + self._summary = self._aggregate(results) + + # generate html report + self.exception_stage = "generate html report" + report.stringify_summary(self._summary) + + if self.save_tests: + utils.dump_json_file( + self._summary, + utils.prepare_log_file_abs_path(self.test_path, "summary.json") + ) + # save variables and export data + vars_out = self.get_vars_out() # TODO + utils.dump_json_file( + vars_out, + utils.prepare_log_file_abs_path(self.test_path, "io.json") + ) + + return self._summary + + def run_path(self, path, dot_env_path=None, mapping=None): + """ run testcase/testsuite file or folder. + + Args: + path (str): testcase/testsuite file/foler path. + dot_env_path (str): specified .env file path. + mapping (dict): if mapping is specified, it will override variables in config block. + + Returns: + dict: result summary + + """ + # load tests + self.exception_stage = "load tests" + tests_mapping = loader.load_cases(path, dot_env_path) + + if mapping: + tests_mapping["project_mapping"]["variables"] = mapping + + return self.run_tests(tests_mapping) + + def run(self, path_or_tests, dot_env_path=None, mapping=None): + """ main interface. + + Args: + path_or_tests: + str: testcase/testsuite file/foler path + dict: valid testcase/testsuite data + dot_env_path (str): specified .env file path. + mapping (dict): if mapping is specified, it will override variables in config block. + + Returns: + dict: result summary + + """ + logger.info(f"HttpRunner version: {__version__}") + if loader.is_test_path(path_or_tests): + return self.run_path(path_or_tests, dot_env_path, mapping) + elif loader.is_test_content(path_or_tests): + project_working_directory = path_or_tests.get("project_mapping", {}).get("PWD", os.getcwd()) + loader.init_pwd(project_working_directory) + return self.run_tests(path_or_tests) + else: + raise exceptions.ParamsError(f"Invalid testcase path or testcases: {path_or_tests}") diff --git a/httprunner/v3/api_test.py b/httprunner/v3/api_test.py new file mode 100644 index 00000000..418c6f15 --- /dev/null +++ b/httprunner/v3/api_test.py @@ -0,0 +1,17 @@ +import unittest + +from httprunner.v3.api import HttpRunner + + +class TestHttpRunner(unittest.TestCase): + + def setUp(self): + self.runner = HttpRunner(failfast=True) + + def test_run_testcase_by_path(self): + summary = self.runner.run_path("examples/postman_echo/request_methods/request_with_variables.yml") + self.assertTrue(summary["success"]) + self.assertEqual(summary["details"][0]["name"], "request methods testcase with variables") + self.assertEqual(summary["details"][0]["records"][0]["name"], "request methods testcase with variables") + self.assertEqual(summary["stat"]["testcases"]["total"], 1) + # self.assertEqual(summary["stat"]["teststeps"]["total"], 2) From 8f3cb320a4772e075ae76880dcfa8015f85ae655 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 22:25:03 +0800 Subject: [PATCH 29/49] feat: html report for v3 --- httprunner/cli.py | 4 +- httprunner/v3/response.py | 99 +++++++++++++++++++++++++++------------ httprunner/v3/runner.py | 10 +++- 3 files changed, 80 insertions(+), 33 deletions(-) diff --git a/httprunner/cli.py b/httprunner/cli.py index 4aac1b87..f828e431 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -7,7 +7,7 @@ if len(sys.argv) >= 2 and sys.argv[1] == "locusts": try: from gevent import monkey monkey.patch_ssl() - from locust.main import main + from locust.main import main as _ except ImportError: msg = """ Locust is not installed, install first and try again. @@ -20,7 +20,7 @@ $ pip install locustio from loguru import logger from httprunner import __description__, __version__ -from httprunner.api import HttpRunner +from httprunner.v3.api import HttpRunner from httprunner.ext.har2case import init_har2case_parser, main_har2case from httprunner.ext.scaffold import init_parser_scaffold, main_scaffold from httprunner.ext.locusts import init_parser_locusts, main_locusts diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py index d4546962..81404b80 100644 --- a/httprunner/v3/response.py +++ b/httprunner/v3/response.py @@ -24,36 +24,7 @@ class ResponseObject(object): "headers": resp_obj.headers, "body": resp_obj.json() } - - def validate(self, - validators: Validators, - variables_mapping: VariablesMapping = None, - functions_mapping: FunctionsMapping = None) -> NoReturn: - - for v in validators: - u_validator = uniform_validator(v) - field = u_validator["check"] - assert_method = u_validator["assert"] - expect_value = u_validator["expect"] - actual_value = jmespath.search(field, self.resp_obj_meta) - - msg = f"assert {field} {assert_method} {expect_value}" - - assert_func = get_mapping_function(assert_method, functions_mapping) - actual_value = parse_string_value(actual_value) - # parse expected value with config/teststep/extracted variables - expect_value = parse_data(expect_value, variables_mapping, functions_mapping) - - try: - assert_func(actual_value, expect_value) - msg += " - success" - logger.info(msg) - except AssertionError: - msg += " - fail" - logger.error(msg) - actual_type = type(actual_value).__name__ - expect_type = type(expect_value).__name__ - raise ValidationFailure(f"assert {field}: {actual_value}({actual_type}) {assert_method} {expect_value}({expect_type})") + self.validation_results = {} def extract(self, extractors: Dict[Text, Text]) -> Dict[Text, Any]: if not extractors: @@ -66,3 +37,71 @@ class ResponseObject(object): logger.info(f"extract mapping: {extract_mapping}") return extract_mapping + + def validate(self, + validators: Validators, + variables_mapping: VariablesMapping = None, + functions_mapping: FunctionsMapping = None) -> NoReturn: + + self.validation_results = {} + if not validators: + return + + validate_pass = True + failures = [] + + for v in validators: + + if "validate_extractor" not in self.validation_results: + self.validation_results["validate_extractor"] = [] + + u_validator = uniform_validator(v) + + # check item + check_item = u_validator["check"] + check_value = jmespath.search(check_item, self.resp_obj_meta) + check_value = parse_string_value(check_value) + + # comparator + assert_method = u_validator["assert"] + assert_func = get_mapping_function(assert_method, functions_mapping) + + # expect item + expect_item = u_validator["expect"] + # parse expected value with config/teststep/extracted variables + expect_value = parse_data(expect_item, variables_mapping, functions_mapping) + + validate_msg = f"assert {check_item} {assert_method} {expect_value}({type(expect_value).__name__})" + + validator_dict = { + "comparator": assert_method, + "check": check_item, + "check_value": check_value, + "expect": expect_item, + "expect_value": expect_value + } + + try: + assert_func(check_value, expect_value) + validate_msg += "\t==> pass" + logger.info(validate_msg) + validator_dict["check_result"] = "pass" + except AssertionError: + validate_pass = False + validator_dict["check_result"] = "fail" + validate_msg += "\t==> fail" + validate_msg += "\n{}({}) {} {}({})".format( + check_value, + type(check_value).__name__, + assert_method, + expect_value, + type(expect_value).__name__ + ) + logger.error(validate_msg) + failures.append(validate_msg) + + self.validation_results["validate_extractor"].append(validator_dict) + + if not validate_pass: + failures_string = "\n".join([failure for failure in failures]) + raise ValidationFailure(failures_string) diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index bbf2a30f..1225e795 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -3,6 +3,7 @@ from typing import List from loguru import logger from httprunner.client import HttpSession +from httprunner.v3.exceptions import ValidationFailure from httprunner.v3.parser import build_url, parse_data, parse_variables_mapping from httprunner.v3.response import ResponseObject from httprunner.v3.schema import TestsConfig, TestStep, VariablesMapping, TestCase @@ -14,6 +15,7 @@ class TestCaseRunner(object): teststeps: List[TestStep] = [] session: HttpSession = None meta_datas: List = [] + validation_results: List = [] def init(self, testcase: TestCase) -> "TestCaseRunner": self.config = testcase.config @@ -59,7 +61,12 @@ class TestCaseRunner(object): # validate validators = step.validators - resp_obj.validate(validators, variables_mapping, self.config.functions) + try: + resp_obj.validate(validators, variables_mapping, self.config.functions) + except ValidationFailure: + raise + finally: + self.validation_results = resp_obj.validation_results return extract_mapping @@ -79,6 +86,7 @@ class TestCaseRunner(object): # save extracted variables to session variables session_variables.update(extract_mapping) # save request & response meta data + self.session.meta_data["validators"] = self.validation_results self.session.meta_data["name"] = step.name self.meta_datas.append(self.session.meta_data) From 3a012f012e8924abb4a4e6cea66d5530b48163e6 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 22:42:14 +0800 Subject: [PATCH 30/49] feat: log detailed request & response when failed --- httprunner/v3/response.py | 31 +++++++++++++++++++++++-------- httprunner/v3/runner.py | 24 ++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py index 81404b80..ce4dde7c 100644 --- a/httprunner/v3/response.py +++ b/httprunner/v3/response.py @@ -4,7 +4,7 @@ import jmespath import requests from loguru import logger -from httprunner.v3.exceptions import ValidationFailure +from httprunner.v3.exceptions import ValidationFailure, ParamsError from httprunner.v3.parser import parse_data, parse_string_value, get_mapping_function from httprunner.v3.schema import VariablesMapping, Validators, FunctionsMapping from httprunner.v3.validator import uniform_validator @@ -19,6 +19,7 @@ class ResponseObject(object): resp_obj (instance): requests.Response instance """ + self.resp_obj = resp_obj self.resp_obj_meta = { "status_code": resp_obj.status_code, "headers": resp_obj.headers, @@ -26,6 +27,22 @@ class ResponseObject(object): } self.validation_results = {} + def __getattr__(self, key): + try: + if key == "json": + value = self.resp_obj.json() + elif key == "cookies": + value = self.resp_obj.cookies.get_dict() + else: + value = getattr(self.resp_obj, key) + + self.__dict__[key] = value + return value + except AttributeError: + err_msg = f"ResponseObject does not have attribute: {key}" + logger.error(err_msg) + raise ParamsError(err_msg) + def extract(self, extractors: Dict[Text, Text]) -> Dict[Text, Any]: if not extractors: return {} @@ -90,13 +107,11 @@ class ResponseObject(object): validate_pass = False validator_dict["check_result"] = "fail" validate_msg += "\t==> fail" - validate_msg += "\n{}({}) {} {}({})".format( - check_value, - type(check_value).__name__, - assert_method, - expect_value, - type(expect_value).__name__ - ) + validate_msg += f"\n" \ + f"check_item: {check_item}\n" \ + f"check_value: {check_value}({type(check_value).__name__})\n" \ + f"assert_method: {assert_method}\n" \ + f"expect_value: {expect_value}({type(expect_value).__name__})" logger.error(validate_msg) failures.append(validate_msg) diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 1225e795..2b0be8c9 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -2,6 +2,7 @@ from typing import List from loguru import logger +from httprunner import utils from httprunner.client import HttpSession from httprunner.v3.exceptions import ValidationFailure from httprunner.v3.parser import build_url, parse_data, parse_variables_mapping @@ -52,6 +53,28 @@ class TestCaseRunner(object): resp = self.session.request(method, url, **parsed_request_dict) resp_obj = ResponseObject(resp) + def log_req_resp_details(): + err_msg = "\n{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32) + + # log request + err_msg += "====== request details ======\n" + err_msg += f"url: {url}\n" + err_msg += f"method: {method}\n" + headers = parsed_request_dict.pop("headers", {}) + err_msg += f"headers: {headers}\n" + for k, v in parsed_request_dict.items(): + v = utils.omit_long_data(v) + err_msg += f"{k}: {repr(v)}\n" + + err_msg += "\n" + + # log response + err_msg += "====== response details ======\n" + err_msg += f"status_code: {resp_obj.status_code}\n" + err_msg += f"headers: {resp_obj.headers}\n" + err_msg += f"body: {repr(resp_obj.text)}\n" + logger.error(err_msg) + # extract extractors = step.extract extract_mapping = resp_obj.extract(extractors) @@ -64,6 +87,7 @@ class TestCaseRunner(object): try: resp_obj.validate(validators, variables_mapping, self.config.functions) except ValidationFailure: + log_req_resp_details() raise finally: self.validation_results = resp_obj.validation_results From d92c17abd45eef0f1dfc1c096fca0fc9d0bd970e Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 22:51:38 +0800 Subject: [PATCH 31/49] fix: log request & response meta data when test failed --- httprunner/v3/runner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 2b0be8c9..33b2e1f5 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -91,6 +91,10 @@ class TestCaseRunner(object): raise finally: self.validation_results = resp_obj.validation_results + # save request & response meta data + self.session.meta_data["validators"] = self.validation_results + self.session.meta_data["name"] = step.name + self.meta_datas.append(self.session.meta_data) return extract_mapping @@ -109,10 +113,6 @@ class TestCaseRunner(object): extract_mapping = self.__run_step(step) # save extracted variables to session variables session_variables.update(extract_mapping) - # save request & response meta data - self.session.meta_data["validators"] = self.validation_results - self.session.meta_data["name"] = step.name - self.meta_datas.append(self.session.meta_data) return self From 499ec50886fd7eebe36e4fae14f0790e14cd4c98 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 23:01:13 +0800 Subject: [PATCH 32/49] change: update schema --- httprunner/v3/{schema/__init__.py => schema.py} | 1 - 1 file changed, 1 deletion(-) rename httprunner/v3/{schema/__init__.py => schema.py} (98%) diff --git a/httprunner/v3/schema/__init__.py b/httprunner/v3/schema.py similarity index 98% rename from httprunner/v3/schema/__init__.py rename to httprunner/v3/schema.py index 5d75ac71..41721a60 100644 --- a/httprunner/v3/schema/__init__.py +++ b/httprunner/v3/schema.py @@ -57,7 +57,6 @@ class Request(BaseModel): class TestStep(BaseModel): name: Name - times: int = 1 request: Request variables: VariablesMapping = {} extract: Dict[Text, Text] = {} From 47ed8c086dd1060e68c2face709093c5b6efc9a8 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 23:08:26 +0800 Subject: [PATCH 33/49] change cli: run_path --- httprunner/cli.py | 2 +- httprunner/v3/api.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/httprunner/cli.py b/httprunner/cli.py index f828e431..671969a3 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -73,7 +73,7 @@ def main_run(args): err_code = 0 try: for path in args.testfile_paths: - summary = runner.run(path, dot_env_path=args.dot_env_path) + summary = runner.run_path(path, dot_env_path=args.dot_env_path) report_dir = args.report_dir or os.path.join(os.getcwd(), "reports") gen_html_report( summary, diff --git a/httprunner/v3/api.py b/httprunner/v3/api.py index e77f8a30..725dffc0 100644 --- a/httprunner/v3/api.py +++ b/httprunner/v3/api.py @@ -223,6 +223,7 @@ class HttpRunner(object): """ # load tests + logger.info(f"HttpRunner version: {__version__}") self.exception_stage = "load tests" tests_mapping = loader.load_cases(path, dot_env_path) @@ -245,7 +246,6 @@ class HttpRunner(object): dict: result summary """ - logger.info(f"HttpRunner version: {__version__}") if loader.is_test_path(path_or_tests): return self.run_path(path_or_tests, dot_env_path, mapping) elif loader.is_test_content(path_or_tests): From 5478fbe9b2937ab83bb49053c727d5016d8c3607 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 21 Apr 2020 23:27:18 +0800 Subject: [PATCH 34/49] change: remove unused code --- httprunner/v3/exceptions/__init__.py | 81 ---------------------------- httprunner/v3/parser.py | 2 +- httprunner/v3/parser_test.py | 2 +- httprunner/v3/response.py | 2 +- httprunner/v3/runner.py | 2 +- httprunner/v3/validator.py | 2 +- 6 files changed, 5 insertions(+), 86 deletions(-) delete mode 100644 httprunner/v3/exceptions/__init__.py diff --git a/httprunner/v3/exceptions/__init__.py b/httprunner/v3/exceptions/__init__.py deleted file mode 100644 index 77d1be52..00000000 --- a/httprunner/v3/exceptions/__init__.py +++ /dev/null @@ -1,81 +0,0 @@ -""" failure type exceptions - these exceptions will mark test as failure -""" - - -class MyBaseFailure(Exception): - pass - - -class ParseTestsFailure(MyBaseFailure): - pass - - -class ValidationFailure(MyBaseFailure): - pass - - -class ExtractFailure(MyBaseFailure): - pass - - -class SetupHooksFailure(MyBaseFailure): - pass - - -class TeardownHooksFailure(MyBaseFailure): - pass - - -""" error type exceptions - these exceptions will mark test as error -""" - - -class MyBaseError(Exception): - pass - - -class FileFormatError(MyBaseError): - pass - - -class ParamsError(MyBaseError): - pass - - -class NotFoundError(MyBaseError): - pass - - -class FileNotFound(FileNotFoundError, NotFoundError): - pass - - -class FunctionNotFound(NotFoundError): - pass - - -class VariableNotFound(NotFoundError): - pass - - -class EnvNotFound(NotFoundError): - pass - - -class CSVNotFound(NotFoundError): - pass - - -class ApiNotFound(NotFoundError): - pass - - -class TestcaseNotFound(NotFoundError): - pass - - -class SummaryEmpty(MyBaseError): - """ test result summary data is empty - """ diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index c0e12c7a..12aa9e7b 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -4,7 +4,7 @@ import re from typing import Any, Set, Text, Callable, List, Dict from httprunner import loader, utils -from httprunner.v3 import exceptions +from httprunner import exceptions from httprunner.v3.schema import VariablesMapping, FunctionsMapping absolute_http_url_regexp = re.compile(r"^https?://", re.I) diff --git a/httprunner/v3/parser_test.py b/httprunner/v3/parser_test.py index ee42c525..9d9ec9e6 100644 --- a/httprunner/v3/parser_test.py +++ b/httprunner/v3/parser_test.py @@ -2,7 +2,7 @@ import time import unittest from httprunner.v3 import parser -from httprunner.v3.exceptions import VariableNotFound, FunctionNotFound +from httprunner.exceptions import VariableNotFound, FunctionNotFound class TestParserBasic(unittest.TestCase): diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py index ce4dde7c..57f47eae 100644 --- a/httprunner/v3/response.py +++ b/httprunner/v3/response.py @@ -4,7 +4,7 @@ import jmespath import requests from loguru import logger -from httprunner.v3.exceptions import ValidationFailure, ParamsError +from httprunner.exceptions import ValidationFailure, ParamsError from httprunner.v3.parser import parse_data, parse_string_value, get_mapping_function from httprunner.v3.schema import VariablesMapping, Validators, FunctionsMapping from httprunner.v3.validator import uniform_validator diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 33b2e1f5..4de890d8 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -4,7 +4,7 @@ from loguru import logger from httprunner import utils from httprunner.client import HttpSession -from httprunner.v3.exceptions import ValidationFailure +from httprunner.exceptions import ValidationFailure from httprunner.v3.parser import build_url, parse_data, parse_variables_mapping from httprunner.v3.response import ResponseObject from httprunner.v3.schema import TestsConfig, TestStep, VariablesMapping, TestCase diff --git a/httprunner/v3/validator.py b/httprunner/v3/validator.py index c1278c2e..3df458f3 100644 --- a/httprunner/v3/validator.py +++ b/httprunner/v3/validator.py @@ -1,6 +1,6 @@ from typing import Text -from httprunner.v3.exceptions import ParamsError +from httprunner.exceptions import ParamsError def get_uniform_comparator(comparator: Text): From 85a1ef27d1548472cf1a2f3254317a65b8e942f1 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 22 Apr 2020 11:58:33 +0800 Subject: [PATCH 35/49] fix --- httprunner/v3/parser.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/httprunner/v3/parser.py b/httprunner/v3/parser.py index 12aa9e7b..d44b6bcd 100644 --- a/httprunner/v3/parser.py +++ b/httprunner/v3/parser.py @@ -3,8 +3,7 @@ import builtins import re from typing import Any, Set, Text, Callable, List, Dict -from httprunner import loader, utils -from httprunner import exceptions +from httprunner import loader, utils, exceptions from httprunner.v3.schema import VariablesMapping, FunctionsMapping absolute_http_url_regexp = re.compile(r"^https?://", re.I) From 0ba3bbb84ffa7a3eef61db94c631fe0a4fdd0ecd Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 22 Apr 2020 19:04:51 +0800 Subject: [PATCH 36/49] refactor: get testcase summary when run suite --- httprunner/v3/api.py | 47 +++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/httprunner/v3/api.py b/httprunner/v3/api.py index 725dffc0..fb1708c9 100644 --- a/httprunner/v3/api.py +++ b/httprunner/v3/api.py @@ -1,7 +1,7 @@ import os import sys import unittest -from typing import List, Tuple +from typing import List, Dict from loguru import logger @@ -91,10 +91,10 @@ class HttpRunner(object): return prepared_testcases - def _run_suite(self, prepared_testcases: List[unittest.TestSuite]) -> List[Tuple]: + def _run_suite(self, prepared_testcases: List[unittest.TestSuite]) -> List[Dict]: """ run prepared testcases """ - tests_results: List[Tuple] = [] + tests_results: List[Dict] = [] for index, testcase in enumerate(prepared_testcases): log_handler = None @@ -107,21 +107,32 @@ class HttpRunner(object): logger.info(f"Start to run testcase: {testcase.config.name}") result = self.unittest_runner.run(testcase) - if result.wasSuccessful(): - tests_results.append((testcase, result)) - else: - tests_results.insert(0, (testcase, result)) + testcase_summary = report.get_summary(result) + testcase_summary["name"] = testcase.config.name + testcase_summary["in_out"] = { + "in": testcase.config.variables, + "out": testcase.config.export + } if self.save_tests and log_handler: logger.remove(log_handler) + logs_file_abs_path = utils.prepare_log_file_abs_path( + self.test_path, f"testcase_{index+1}.log" + ) + testcase_summary["log"] = logs_file_abs_path + + if result.wasSuccessful(): + tests_results.append(testcase_summary) + else: + tests_results.insert(0, testcase_summary) return tests_results - def _aggregate(self, tests_results: List[Tuple]): - """ aggregate results + def _aggregate(self, tests_results: List[Dict]): + """ aggregate multiple testcase results Args: - tests_results (list): list of (testcase, result) + tests_results (list): list of testcase summary """ summary = { @@ -139,31 +150,17 @@ class HttpRunner(object): "details": [] } - for index, tests_result in enumerate(tests_results): - testcase, result = tests_result - testcase_summary = report.get_summary(result) - + for testcase_summary in tests_results: if testcase_summary["success"]: summary["stat"]["testcases"]["success"] += 1 else: summary["stat"]["testcases"]["fail"] += 1 summary["success"] &= testcase_summary["success"] - testcase_summary["name"] = testcase.config.name - testcase_summary["in_out"] = { - "in": testcase.config.variables, - "out": testcase.config.export - } report.aggregate_stat(summary["stat"]["teststeps"], testcase_summary["stat"]) report.aggregate_stat(summary["time"], testcase_summary["time"]) - if self.save_tests: - logs_file_abs_path = utils.prepare_log_file_abs_path( - self.test_path, f"testcase_{index+1}.log" - ) - testcase_summary["log"] = logs_file_abs_path - summary["details"].append(testcase_summary) return summary From 310f1ea30ce5b92b7e0da1e1e28bd02c0e95fb50 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 22 Apr 2020 21:16:58 +0800 Subject: [PATCH 37/49] refactor: add typing with pydantic --- httprunner/cli.py | 6 +-- httprunner/client.py | 36 ++++++-------- httprunner/report/html/gen_report.py | 15 +++--- httprunner/report/html/result.py | 16 ++++--- httprunner/report/html/template.html | 2 +- httprunner/report/stringify.py | 37 +++++++-------- httprunner/report/summarize.py | 46 ++++++++---------- httprunner/v3/api.py | 42 ++++++++-------- httprunner/v3/response.py | 2 +- httprunner/v3/runner.py | 12 ++--- httprunner/v3/schema.py | 71 ++++++++++++++++++++++++++++ 11 files changed, 171 insertions(+), 114 deletions(-) diff --git a/httprunner/cli.py b/httprunner/cli.py index 671969a3..2523c17e 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -73,15 +73,15 @@ def main_run(args): err_code = 0 try: for path in args.testfile_paths: - summary = runner.run_path(path, dot_env_path=args.dot_env_path) + testsuite_summary = runner.run_path(path, dot_env_path=args.dot_env_path) report_dir = args.report_dir or os.path.join(os.getcwd(), "reports") gen_html_report( - summary, + testsuite_summary, report_template=args.report_template, report_dir=report_dir, report_file=args.report_file ) - err_code |= (0 if summary and summary["success"] else 1) + err_code |= (0 if testsuite_summary and testsuite_summary.success else 1) except Exception as ex: logger.error(f"!!!!!!!!!! exception stage: {runner.exception_stage} !!!!!!!!!!\n{str(ex)}") err_code = 1 diff --git a/httprunner/client.py b/httprunner/client.py index 4577a40a..41e12129 100644 --- a/httprunner/client.py +++ b/httprunner/client.py @@ -11,6 +11,7 @@ from requests.exceptions import (InvalidSchema, InvalidURL, MissingSchema, from httprunner import response from httprunner.utils import lower_dict_keys, omit_long_data +from httprunner.v3.schema import MetaData, RequestStat urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -107,9 +108,8 @@ class HttpSession(requests.Session): def init_meta_data(self): """ initialize meta_data, it will store detail data of request and response """ - self.meta_data = { - "name": "", - "data": [ + self.meta_data = MetaData( + data=[ { "request": { "url": "N/A", @@ -124,19 +124,15 @@ class HttpSession(requests.Session): } } ], - "stat": { - "content_size": "N/A", - "response_time_ms": "N/A", - "elapsed_ms": "N/A", - } - } + stat=RequestStat() + ) def update_last_req_resp_record(self, resp_obj): """ update request and response info from Response() object. """ - self.meta_data["data"].pop() - self.meta_data["data"].append(get_req_resp_record(resp_obj)) + self.meta_data.data.pop() + self.meta_data.data.append(get_req_resp_record(resp_obj)) def request(self, method, url, name=None, **kwargs): """ @@ -180,13 +176,13 @@ class HttpSession(requests.Session): self.init_meta_data() # record test name - self.meta_data["name"] = name + self.meta_data.name = name # record original request info - self.meta_data["data"][0]["request"]["method"] = method - self.meta_data["data"][0]["request"]["url"] = url + self.meta_data.data[0]["request"]["method"] = method + self.meta_data.data[0]["request"]["url"] = url kwargs.setdefault("timeout", 120) - self.meta_data["data"][0]["request"].update(kwargs) + self.meta_data.data[0]["request"].update(kwargs) start_timestamp = time.time() response = self._send_request_safe_mode(method, url, **kwargs) @@ -200,15 +196,13 @@ class HttpSession(requests.Session): content_size = len(response.content or "") # record the consumed time - self.meta_data["stat"] = { - "response_time_ms": response_time_ms, - "elapsed_ms": response.elapsed.microseconds / 1000.0, - "content_size": content_size - } + self.meta_data.stat.response_time_ms = response_time_ms + self.meta_data.stat.elapsed_ms = response.elapsed.microseconds / 1000.0 + self.meta_data.stat.content_size = content_size # record request and response histories, include 30X redirection response_list = response.history + [response] - self.meta_data["data"] = [ + self.meta_data.data = [ get_req_resp_record(resp_obj) for resp_obj in response_list ] diff --git a/httprunner/report/html/gen_report.py b/httprunner/report/html/gen_report.py index c7791183..95772079 100644 --- a/httprunner/report/html/gen_report.py +++ b/httprunner/report/html/gen_report.py @@ -6,20 +6,21 @@ from jinja2 import Template from loguru import logger from httprunner.exceptions import SummaryEmpty +from httprunner.v3.schema import TestSuiteSummary -def gen_html_report(summary, report_template=None, report_dir=None, report_file=None): +def gen_html_report(testsuite_summary: TestSuiteSummary, report_template=None, report_dir=None, report_file=None): """ render html report with specified report name and template Args: - summary (dict): test result summary data + testsuite_summary (dict): testsuite result summary data report_template (str): specify html report template path, template should be in Jinja2 format. report_dir (str): specify html report save directory report_file (str): specify html report file path, this has higher priority than specifying report dir. """ - if not summary["time"] or summary["stat"]["testcases"]["total"] == 0: - logger.error(f"test result summary is empty ! {summary}") + if not testsuite_summary.time or testsuite_summary.stat.testcases["total"] == 0: + logger.error(f"test result testsuite_summary is empty ! {testsuite_summary}") raise SummaryEmpty if not report_template: @@ -33,9 +34,9 @@ def gen_html_report(summary, report_template=None, report_dir=None, report_file= logger.info("Start to render Html report ...") - start_at_timestamp = summary["time"]["start_at"] + start_at_timestamp = testsuite_summary.time.start_at utc_time_iso_8601_str = datetime.utcfromtimestamp(start_at_timestamp).isoformat() - summary["time"]["start_datetime"] = utc_time_iso_8601_str + testsuite_summary.time.start_datetime = utc_time_iso_8601_str if report_file: report_dir = os.path.dirname(report_file) @@ -55,7 +56,7 @@ def gen_html_report(summary, report_template=None, report_dir=None, report_file= rendered_content = Template( template_content, extensions=["jinja2.ext.loopcontrols"] - ).render(summary) + ).render(testsuite_summary.dict()) fp_w.write(rendered_content) logger.info(f"Generated Html report: {report_path}") diff --git a/httprunner/report/html/result.py b/httprunner/report/html/result.py index 762d0bb1..e9d88f2f 100644 --- a/httprunner/report/html/result.py +++ b/httprunner/report/html/result.py @@ -3,6 +3,8 @@ import unittest from loguru import logger +from httprunner.v3.schema import Record + class HtmlTestResult(unittest.TextTestResult): """ A html result class that can generate formatted html results. @@ -13,13 +15,13 @@ class HtmlTestResult(unittest.TextTestResult): self.records = [] def _record_test(self, test, status, attachment=''): - data = { - 'name': test.shortDescription(), - 'status': status, - 'attachment': attachment, - "meta_datas": test.meta_datas - } - self.records.append(data) + record = Record( + name=test.shortDescription(), + status=status, + attachment=attachment, + meta_datas=test.meta_datas + ) + self.records.append(record) def startTestRun(self): self.start_at = time.time() diff --git a/httprunner/report/html/template.html b/httprunner/report/html/template.html index 8bbfc1bf..a205a73f 100644 --- a/httprunner/report/html/template.html +++ b/httprunner/report/html/template.html @@ -201,7 +201,7 @@ {% for record in test_suite_summary.records %} {% set record_index = "{}_{}".format(suite_index, loop.index) %} - {% set record_meta_datas = record.meta_datas_expanded %} + {% set record_meta_datas = record.meta_datas %} {{record.status}} {{record.name}} diff --git a/httprunner/report/stringify.py b/httprunner/report/stringify.py index c6b9cf11..d13e2062 100644 --- a/httprunner/report/stringify.py +++ b/httprunner/report/stringify.py @@ -1,10 +1,13 @@ import json from base64 import b64encode from collections import Iterable +from typing import List from jinja2 import escape from requests.cookies import RequestsCookieJar +from httprunner.v3.schema import TestSuiteSummary, MetaData + def dumps_json(value): """ dumps json value to indented string @@ -164,20 +167,20 @@ def __expand_meta_datas(meta_datas, meta_datas_expanded): [dict1, dict2, dict3] """ - if isinstance(meta_datas, dict): + if isinstance(meta_datas, MetaData): meta_datas_expanded.append(meta_datas) elif isinstance(meta_datas, list): for meta_data in meta_datas: __expand_meta_datas(meta_data, meta_datas_expanded) -def __get_total_response_time(meta_datas_expanded): +def __get_total_response_time(meta_datas: List[MetaData]): """ caculate total response time of all meta_datas """ try: response_time = 0 - for meta_data in meta_datas_expanded: - response_time += meta_data["stat"]["response_time_ms"] + for meta_data in meta_datas: + response_time += meta_data.stat.response_time_ms return "{:.2f}".format(response_time) @@ -186,30 +189,24 @@ def __get_total_response_time(meta_datas_expanded): return "N/A" -def __stringify_meta_datas(meta_datas): +def __stringify_meta_datas(meta_datas: List[MetaData]): - if isinstance(meta_datas, list): - for _meta_data in meta_datas: - __stringify_meta_datas(_meta_data) - elif isinstance(meta_datas, dict): - data_list = meta_datas["data"] + for meta_data in meta_datas: + data_list = meta_data.data for data in data_list: __stringify_request(data["request"]) __stringify_response(data["response"]) -def stringify_summary(summary): +def stringify_summary(testsuite_summary: TestSuiteSummary): """ stringify summary, in order to dump json file and generate html report. """ - for index, suite_summary in enumerate(summary["details"]): + for index, testcase_summary in enumerate(testsuite_summary.details): - if not suite_summary.get("name"): - suite_summary["name"] = f"testcase {index}" + if not testcase_summary.name: + testcase_summary.name = f"testcase {index}" - for record in suite_summary.get("records"): - meta_datas = record['meta_datas'] + for record in testcase_summary.records: + meta_datas = record.meta_datas __stringify_meta_datas(meta_datas) - meta_datas_expanded = [] - __expand_meta_datas(meta_datas, meta_datas_expanded) - record["meta_datas_expanded"] = meta_datas_expanded - record["response_time"] = __get_total_response_time(meta_datas_expanded) + record.response_time = __get_total_response_time(meta_datas) diff --git a/httprunner/report/summarize.py b/httprunner/report/summarize.py index 93c7145f..404d8cb4 100644 --- a/httprunner/report/summarize.py +++ b/httprunner/report/summarize.py @@ -1,6 +1,8 @@ import platform from httprunner import __version__ +from httprunner.report.html.result import HtmlTestResult +from httprunner.v3.schema import TestCaseSummary, TestCaseStat, TestCaseTime, TestCaseInOut def get_platform(): @@ -38,7 +40,7 @@ def aggregate_stat(origin_stat, new_stat): origin_stat[key] += new_stat[key] -def get_summary(result): +def get_summary(result: HtmlTestResult) -> TestCaseSummary: """ get summary from test result Args: @@ -55,28 +57,20 @@ def get_summary(result): } """ - summary = { - "success": result.wasSuccessful(), - "stat": { - 'total': result.testsRun, - 'failures': len(result.failures), - 'errors': len(result.errors), - 'skipped': len(result.skipped), - 'expectedFailures': len(result.expectedFailures), - 'unexpectedSuccesses': len(result.unexpectedSuccesses) - } - } - summary["stat"]["successes"] = summary["stat"]["total"] \ - - summary["stat"]["failures"] \ - - summary["stat"]["errors"] \ - - summary["stat"]["skipped"] \ - - summary["stat"]["expectedFailures"] \ - - summary["stat"]["unexpectedSuccesses"] - - summary["time"] = { - 'start_at': result.start_at, - 'duration': result.duration - } - summary["records"] = result.records - - return summary + return TestCaseSummary( + success=result.wasSuccessful(), + stat=TestCaseStat( + total=result.testsRun, + failures=len(result.failures), + errors=len(result.errors), + skipped=len(result.skipped), + expectedFailures=len(result.expectedFailures), + unexpectedSuccesses=len(result.unexpectedSuccesses) + ), + time=TestCaseTime( + start_at=result.start_at, + duration=result.duration + ), + records=result.records, + in_out=TestCaseInOut() + ) diff --git a/httprunner/v3/api.py b/httprunner/v3/api.py index fb1708c9..b7a11d63 100644 --- a/httprunner/v3/api.py +++ b/httprunner/v3/api.py @@ -7,7 +7,7 @@ from loguru import logger from httprunner import report, loader, utils, exceptions, __version__ from httprunner.v3.runner import TestCaseRunner -from httprunner.v3.schema import TestsMapping +from httprunner.v3.schema import TestsMapping, TestCaseSummary, TestSuiteSummary class HttpRunner(object): @@ -91,10 +91,10 @@ class HttpRunner(object): return prepared_testcases - def _run_suite(self, prepared_testcases: List[unittest.TestSuite]) -> List[Dict]: + def _run_suite(self, prepared_testcases: List[unittest.TestSuite]) -> List[TestCaseSummary]: """ run prepared testcases """ - tests_results: List[Dict] = [] + tests_results: List[TestCaseSummary] = [] for index, testcase in enumerate(prepared_testcases): log_handler = None @@ -108,18 +108,16 @@ class HttpRunner(object): result = self.unittest_runner.run(testcase) testcase_summary = report.get_summary(result) - testcase_summary["name"] = testcase.config.name - testcase_summary["in_out"] = { - "in": testcase.config.variables, - "out": testcase.config.export - } + testcase_summary.name = testcase.config.name + testcase_summary.in_out.vars = testcase.config.variables + testcase_summary.in_out.out = testcase.config.export if self.save_tests and log_handler: logger.remove(log_handler) logs_file_abs_path = utils.prepare_log_file_abs_path( self.test_path, f"testcase_{index+1}.log" ) - testcase_summary["log"] = logs_file_abs_path + testcase_summary.log = logs_file_abs_path if result.wasSuccessful(): tests_results.append(testcase_summary) @@ -128,14 +126,14 @@ class HttpRunner(object): return tests_results - def _aggregate(self, tests_results: List[Dict]): + def _aggregate(self, tests_results: List[TestCaseSummary]) -> TestSuiteSummary: """ aggregate multiple testcase results Args: tests_results (list): list of testcase summary """ - summary = { + testsuite_summary = { "success": True, "stat": { "testcases": { @@ -151,21 +149,21 @@ class HttpRunner(object): } for testcase_summary in tests_results: - if testcase_summary["success"]: - summary["stat"]["testcases"]["success"] += 1 + if testcase_summary.success: + testsuite_summary["stat"]["testcases"]["success"] += 1 else: - summary["stat"]["testcases"]["fail"] += 1 + testsuite_summary["stat"]["testcases"]["fail"] += 1 - summary["success"] &= testcase_summary["success"] + testsuite_summary["success"] &= testcase_summary.success - report.aggregate_stat(summary["stat"]["teststeps"], testcase_summary["stat"]) - report.aggregate_stat(summary["time"], testcase_summary["time"]) + report.aggregate_stat(testsuite_summary["stat"]["teststeps"], testcase_summary.stat.dict()) + report.aggregate_stat(testsuite_summary["time"], testcase_summary.time.dict()) - summary["details"].append(testcase_summary) + testsuite_summary["details"].append(testcase_summary) - return summary + return TestSuiteSummary.parse_obj(testsuite_summary) - def run_tests(self, tests_mapping): + def run_tests(self, tests_mapping) -> TestSuiteSummary: """ run testcase/testsuite data """ tests = TestsMapping.parse_obj(tests_mapping) @@ -195,7 +193,7 @@ class HttpRunner(object): if self.save_tests: utils.dump_json_file( - self._summary, + self._summary.dict(), utils.prepare_log_file_abs_path(self.test_path, "summary.json") ) # save variables and export data @@ -207,7 +205,7 @@ class HttpRunner(object): return self._summary - def run_path(self, path, dot_env_path=None, mapping=None): + def run_path(self, path, dot_env_path=None, mapping=None) -> TestSuiteSummary: """ run testcase/testsuite file or folder. Args: diff --git a/httprunner/v3/response.py b/httprunner/v3/response.py index 57f47eae..01af0bd8 100644 --- a/httprunner/v3/response.py +++ b/httprunner/v3/response.py @@ -25,7 +25,7 @@ class ResponseObject(object): "headers": resp_obj.headers, "body": resp_obj.json() } - self.validation_results = {} + self.validation_results: Dict = {} def __getattr__(self, key): try: diff --git a/httprunner/v3/runner.py b/httprunner/v3/runner.py index 4de890d8..b426b2ea 100644 --- a/httprunner/v3/runner.py +++ b/httprunner/v3/runner.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Dict from loguru import logger @@ -7,7 +7,7 @@ from httprunner.client import HttpSession from httprunner.exceptions import ValidationFailure from httprunner.v3.parser import build_url, parse_data, parse_variables_mapping from httprunner.v3.response import ResponseObject -from httprunner.v3.schema import TestsConfig, TestStep, VariablesMapping, TestCase +from httprunner.v3.schema import TestsConfig, TestStep, VariablesMapping, TestCase, MetaData class TestCaseRunner(object): @@ -15,8 +15,8 @@ class TestCaseRunner(object): config: TestsConfig = {} teststeps: List[TestStep] = [] session: HttpSession = None - meta_datas: List = [] - validation_results: List = [] + meta_datas: List[MetaData] = [] + validation_results: Dict = {} def init(self, testcase: TestCase) -> "TestCaseRunner": self.config = testcase.config @@ -92,8 +92,8 @@ class TestCaseRunner(object): finally: self.validation_results = resp_obj.validation_results # save request & response meta data - self.session.meta_data["validators"] = self.validation_results - self.session.meta_data["name"] = step.name + self.session.meta_data.validators = self.validation_results + self.session.meta_data.name = step.name self.meta_datas.append(self.session.meta_data) return extract_mapping diff --git a/httprunner/v3/schema.py b/httprunner/v3/schema.py index 41721a60..de2f144c 100644 --- a/httprunner/v3/schema.py +++ b/httprunner/v3/schema.py @@ -80,3 +80,74 @@ class ProjectMeta(BaseModel): class TestsMapping(BaseModel): project_mapping: ProjectMeta # TODO: rename to project_meta testcases: List[TestCase] + + +class Stat(BaseModel): + testcases: Dict + teststeps: Dict + + +class TestCaseTime(BaseModel): + start_at: float + duration: float + start_datetime: Text = "" + + +class TestCaseStat(BaseModel): + total: int = 0 + successes: int = 0 + failures: int = 0 + errors: int = 0 + skipped: int = 0 + expectedFailures: int = 0 + unexpectedSuccesses: int = 0 + + +class TestCaseInOut(BaseModel): + vars: VariablesMapping = {} + out: Export = [] + + +class RequestStat(BaseModel): + content_size: Text = "N/A" + response_time_ms: Text = "N/A" + elapsed_ms: Text = "N/A" + + +class MetaData(BaseModel): + name: Text = "" + data: List[Dict] + stat: RequestStat + validators: Dict = {} + + +class Record(BaseModel): + name: Text = "" + status: Text = "" + attachment: Text = "" + meta_datas: List[MetaData] = [] + response_time: Text = "N/A" + + +class TestCaseSummary(BaseModel): + name: Text = "" + success: bool + stat: TestCaseStat + time: TestCaseTime + records: List = [Record] + in_out: TestCaseInOut = {} + log: Text = "" + + +class PlatformInfo(BaseModel): + httprunner_version: Text + python_version: Text + platform: Text + + +class TestSuiteSummary(BaseModel): + success: bool + stat: Stat + time: TestCaseTime + platform: PlatformInfo + details: List[TestCaseSummary] From 3df31fb0c940bad20bd55505c654326a35eba6e1 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 22 Apr 2020 22:13:11 +0800 Subject: [PATCH 38/49] fix html report --- httprunner/report/html/result.py | 17 +++++------- httprunner/report/html/template.html | 39 ++++++++++++---------------- httprunner/report/stringify.py | 8 +++--- httprunner/report/summarize.py | 4 +-- httprunner/v3/api_test.py | 12 ++++----- httprunner/v3/schema.py | 2 +- 6 files changed, 36 insertions(+), 46 deletions(-) diff --git a/httprunner/report/html/result.py b/httprunner/report/html/result.py index e9d88f2f..9cb16ab3 100644 --- a/httprunner/report/html/result.py +++ b/httprunner/report/html/result.py @@ -7,21 +7,18 @@ from httprunner.v3.schema import Record class HtmlTestResult(unittest.TextTestResult): - """ A html result class that can generate formatted html results. - Used by TextTestRunner. + """ A html result class that can generate formatted html results, used by TextTestRunner. + Each testcase is corresponding to one HtmlTestResult instance """ def __init__(self, stream, descriptions, verbosity): super(HtmlTestResult, self).__init__(stream, descriptions, verbosity) - self.records = [] + self.record = Record() def _record_test(self, test, status, attachment=''): - record = Record( - name=test.shortDescription(), - status=status, - attachment=attachment, - meta_datas=test.meta_datas - ) - self.records.append(record) + self.record.name = test.shortDescription() + self.record.status = status + self.record.attachment = attachment + self.record.meta_datas = test.meta_datas def startTestRun(self): self.start_at = time.time() diff --git a/httprunner/report/html/template.html b/httprunner/report/html/template.html index a205a73f..9492e42c 100644 --- a/httprunner/report/html/template.html +++ b/httprunner/report/html/template.html @@ -181,17 +181,10 @@

Details

- {% for test_suite_summary in details %} - {% set suite_index = loop.index %} -

{{test_suite_summary.name}}

- - - - - - - - + {% for testcase_summary in details %} + {% set testcase_index = loop.index %} +

{{testcase_summary.name}}

+
TOTAL: {{test_suite_summary.stat.total}}SUCCESS: {{test_suite_summary.stat.successes}}FAILED: {{test_suite_summary.stat.failures}}ERROR: {{test_suite_summary.stat.errors}}SKIPPED: {{test_suite_summary.stat.skipped}}
@@ -199,22 +192,22 @@ - {% for record in test_suite_summary.records %} - {% set record_index = "{}_{}".format(suite_index, loop.index) %} - {% set record_meta_datas = record.meta_datas %} - + {% set record_meta_datas = testcase_summary.record.meta_datas %} + {% for meta_data in record_meta_datas %} + {% set step_index = "{}_{}".format(testcase_index, loop.index) %} +
Status NameDetail
{{record.status}} {{record.name}} {{ record.response_time }} ms - {% for meta_data in record_meta_datas %} - {% set meta_data_index = "{}_{}".format(record_index, loop.index) %} - log-{{loop.index}} -