diff --git a/examples/HelloWorld/README.md b/examples/HelloWorld/README.md deleted file mode 100644 index f7efe8a2..00000000 --- a/examples/HelloWorld/README.md +++ /dev/null @@ -1,98 +0,0 @@ -# Hello World - -This example shows you how to organize and run testcases in layer. - -## file structure - -According to rules, all testcase definition files should be placed in `tests` folder, and testing reports will be generated in `reports` folder. - -```text -$ cd httprunner/examples/HelloWorld -$ tree . -. -├── README.md -├── reports -│ └── smoketest -│ └── 2018-02-09-16-25-54.html -└── tests - ├── __init__.py - ├── api - │ └── basic.yml - ├── debugtalk.py - ├── suite - │ ├── create_and_check.yml - │ └── setup.yml - └── testcases - └── smoketest.yml -``` - -## Start server - -In order to run test, we need a backend service, and here we will use `api_server` located in our unittests. - -```bash -$ cd httprunner -$ export FLASK_APP=tests/api_server.py -$ flask run - * Serving Flask app "tests.api_server" - * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit) -``` - -## run testcases - -When you want to run testcases, you should make sure you are in the root directory of your project. In this example, that is the HelloWorld folder path. - -```bash -$ cd httprunner/examples/HelloWorld -``` - -Then, run the testcase with `hrun` command. - -``` -$ hrun tests/testcases/smoketest.yml -Running tests... ----------------------------------------------------------------------- - get token ... INFO:root: Start to POST http://127.0.0.1:5000/api/get-token -INFO:root: status_code: 200, response_time: 12 ms, response_length: 46 bytes -OK (0.018492)s - reset all users ... INFO:root: Start to GET http://127.0.0.1:5000/api/reset-all -INFO:root: status_code: 200, response_time: 5 ms, response_length: 17 bytes -OK (0.006153)s - make sure user 1000 does not exist ... INFO:root: Start to GET http://127.0.0.1:5000/api/users/1000 -ERROR:root: Failed to GET http://127.0.0.1:5000/api/users/1000! exception msg: 404 Client Error: NOT FOUND for url: http://127.0.0.1:5000/api/users/1000 -OK (0.010638)s - create user 1000 ... INFO:root: Start to POST http://127.0.0.1:5000/api/users/1000 -INFO:root: status_code: 201, response_time: 9 ms, response_length: 54 bytes -OK (0.010303)s - check if user 1000 exists ... INFO:root: Start to GET http://127.0.0.1:5000/api/users/1000 -INFO:root: status_code: 200, response_time: 11 ms, response_length: 66 bytes -OK (0.013168)s - make sure user 1001 does not exist ... INFO:root: Start to GET http://127.0.0.1:5000/api/users/1001 -ERROR:root: Failed to GET http://127.0.0.1:5000/api/users/1001! exception msg: 404 Client Error: NOT FOUND for url: http://127.0.0.1:5000/api/users/1001 -OK (0.013631)s - create user 1001 ... INFO:root: Start to POST http://127.0.0.1:5000/api/users/1001 -INFO:root: status_code: 201, response_time: 6 ms, response_length: 54 bytes -OK (0.007490)s - check if user 1001 exists ... INFO:root: Start to GET http://127.0.0.1:5000/api/users/1001 -INFO:root: status_code: 200, response_time: 9 ms, response_length: 66 bytes -OK (0.011435)s - ----------------------------------------------------------------------- -Ran 8 tests in 0.092s - -OK - - - -Generating HTML reports... -Template is not specified, load default template instead. -Reports generated: /Users/debugtalk/MyProjects/HttpRunner-dev/HttpRunner/examples/HelloWorld/reports/smoketest/2018-02-09-16-38-14.html -``` - -After the running is over, you will get a testing report, which is in HTML format. - -```bash -$ open reports/smoketest/2018-02-09-16-38-14.html -``` - -![](test-report.jpg) diff --git a/examples/HelloWorld/reports/smoketest/2018-02-09-16-25-54.html b/examples/HelloWorld/reports/smoketest/2018-02-09-16-25-54.html deleted file mode 100644 index 86ad6698..00000000 --- a/examples/HelloWorld/reports/smoketest/2018-02-09-16-25-54.html +++ /dev/null @@ -1,205 +0,0 @@ - - - - Test Result - - - - - -
-
-
-

Test Result

-

Start Time: 2018-02-09 16:25:53

-

Duration: 0.141s

-

Status: Pass: 11

-
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
smoketestStatus
get token - - - Pass - - - -
reset all users - - - Pass - - - -
make sure user 1000 does not exist - - - Pass - - - -
create user 1000 - - - Pass - - - -
check if user 1000 exists - - - Pass - - - -
make sure user 1001 does not exist - - - Pass - - - -
create user 1001 - - - Pass - - - -
check if user 1001 exists - - - Pass - - - -
make sure user 1002 does not exist - - - Pass - - - -
create user 1002 - - - Pass - - - -
check if user 1002 exists - - - Pass - - - -
- Total Test Runned: 11 - - Pass: 11 -
-
-
-
- - - - \ No newline at end of file diff --git a/examples/HelloWorld/reports/smoketest/2018-02-09-16-38-14.html b/examples/HelloWorld/reports/smoketest/2018-02-09-16-38-14.html deleted file mode 100644 index 8df2409b..00000000 --- a/examples/HelloWorld/reports/smoketest/2018-02-09-16-38-14.html +++ /dev/null @@ -1,166 +0,0 @@ - - - - Test Result - - - - - -
-
-
-

Test Result

-

Start Time: 2018-02-09 16:38:14

-

Duration: 0.092s

-

Status: Pass: 8

-
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
smoketestStatus
get token - - - Pass - - - -
reset all users - - - Pass - - - -
make sure user 1000 does not exist - - - Pass - - - -
create user 1000 - - - Pass - - - -
check if user 1000 exists - - - Pass - - - -
make sure user 1001 does not exist - - - Pass - - - -
create user 1001 - - - Pass - - - -
check if user 1001 exists - - - Pass - - - -
- Total Test Runned: 8 - - Pass: 8 -
-
-
-
- - - - \ No newline at end of file diff --git a/examples/HelloWorld/test-report.jpg b/examples/HelloWorld/test-report.jpg deleted file mode 100644 index 245a4db6..00000000 Binary files a/examples/HelloWorld/test-report.jpg and /dev/null differ diff --git a/examples/HelloWorld/tests/__init__.py b/examples/HelloWorld/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/HelloWorld/tests/debugtalk.py b/examples/HelloWorld/tests/debugtalk.py deleted file mode 100644 index c215984d..00000000 --- a/examples/HelloWorld/tests/debugtalk.py +++ /dev/null @@ -1,28 +0,0 @@ -import hashlib -import hmac -import random -import string - -SECRET_KEY = "DebugTalk" -default_request = { - "base_url": "http://127.0.0.1:5000", - "headers": { - "Content-Type": "application/json", - "device_sn": "$device_sn" - } -} - -def gen_random_string(str_len): - random_char_list = [] - for _ in range(str_len): - random_char = random.choice(string.ascii_letters + string.digits) - random_char_list.append(random_char) - - random_string = ''.join(random_char_list) - return random_string - -def get_sign(*args): - content = ''.join(args).encode('ascii') - sign_key = SECRET_KEY.encode('ascii') - sign = hmac.new(sign_key, content, hashlib.sha1).hexdigest() - return sign diff --git a/httprunner/__about__.py b/httprunner/__about__.py index d3464a1b..a1b97cc9 100644 --- a/httprunner/__about__.py +++ b/httprunner/__about__.py @@ -1,7 +1,7 @@ __title__ = 'HttpRunner' __description__ = 'One-stop solution for HTTP(S) testing.' __url__ = 'https://github.com/HttpRunner/HttpRunner' -__version__ = '1.3.12' +__version__ = '1.4.0.beta' __author__ = 'debugtalk' __author_email__ = 'mail@debugtalk.com' __license__ = 'MIT' diff --git a/httprunner/locusts.py b/httprunner/locusts.py index 3d7996fa..25ec95fa 100644 --- a/httprunner/locusts.py +++ b/httprunner/locusts.py @@ -6,8 +6,7 @@ import os import sys from httprunner.logger import color_print -from httprunner.testcase import load_test_file - +from httprunner.testcase import TestcaseLoader from locust.main import main @@ -41,7 +40,7 @@ def gen_locustfile(testcase_file_path): "templates", "locustfile_template" ) - testset = load_test_file(testcase_file_path) + testset = TestcaseLoader.load_test_file(testcase_file_path) host = testset.get("config", {}).get("request", {}).get("base_url", "") with io.open(template_path, encoding='utf-8') as template: diff --git a/httprunner/runner.py b/httprunner/runner.py index e2677ab8..1c264c2b 100644 --- a/httprunner/runner.py +++ b/httprunner/runner.py @@ -2,10 +2,11 @@ from unittest.case import SkipTest -from httprunner import exception, logger, response, testcase, utils +from httprunner import exception, logger, response, utils from httprunner.client import HttpSession from httprunner.context import Context from httprunner.events import EventHook +from httprunner.testcase import TestcaseLoader class Runner(object): @@ -13,7 +14,7 @@ class Runner(object): def __init__(self, config_dict=None, http_client_session=None): self.http_client_session = http_client_session self.context = Context() - testcase.load_test_dependencies() + TestcaseLoader.load_test_dependencies() config_dict = config_dict or {} self.init_config(config_dict, "testset") diff --git a/httprunner/task.py b/httprunner/task.py index 8e2f5e18..6f1067fc 100644 --- a/httprunner/task.py +++ b/httprunner/task.py @@ -7,6 +7,7 @@ import unittest from httprunner import exception, logger, runner, testcase, utils from httprunner.compat import is_py3 from httprunner.report import HtmlTestResult, get_summary, render_html_report +from httprunner.testcase import TestcaseLoader class TestCase(unittest.TestCase): @@ -195,7 +196,7 @@ def init_task_suite(path_or_testsets, mapping=None, http_client_session=None): """ initialize task suite """ if not testcase.is_testsets(path_or_testsets): - testsets = testcase.load_testsets_by_path(path_or_testsets) + testsets = TestcaseLoader.load_testsets_by_path(path_or_testsets) else: testsets = path_or_testsets diff --git a/httprunner/testcase.py b/httprunner/testcase.py index c380eb48..8185844b 100644 --- a/httprunner/testcase.py +++ b/httprunner/testcase.py @@ -10,18 +10,12 @@ import random import re from httprunner import exception, logger, utils -from httprunner.utils import FileUtils from httprunner.compat import OrderedDict, numeric_types +from httprunner.utils import FileUtils variable_regexp = r"\$([\w_]+)" function_regexp = r"\$\{([\w_]+\([\$\w\.\-_ =,]*\))\}" function_regexp_compile = re.compile(r"^([\w_]+)\(([\$\w\.\-_ =,]*)\)$") -test_def_overall_dict = { - "loaded": False, - "api": {}, - "suite": {} -} -testcases_cache_mapping = {} def extract_variables(content): @@ -86,6 +80,8 @@ def parse_function(content): "kwargs": {} } matched = function_regexp_compile.match(content) + if not matched: + raise exception.ApiNotFound("{} not found!".format(content)) function_meta["func_name"] = matched.group(1) args_str = matched.group(2).replace(" ", "") @@ -102,86 +98,291 @@ def parse_function(content): return function_meta -def load_test_dependencies(): - """ load all api and suite definitions. - default api folder is "$CWD/tests/api/". - default suite folder is "$CWD/tests/suite/". - """ - test_def_overall_dict["loaded"] = True - test_def_overall_dict["api"] = {} - test_def_overall_dict["suite"] = {} - # load api definitions - api_def_folder = os.path.join(os.getcwd(), "tests", "api") - api_files = FileUtils.load_folder_files(api_def_folder) +class TestcaseLoader(object): - for test_file in api_files: - testset = load_test_file(test_file) - test_def_overall_dict["api"].update(testset["api"]) + overall_def_dict = { + "api": {}, + "suite": {} + } + testcases_cache_mapping = {} - # load suite definitions - suite_def_folder = os.path.join(os.getcwd(), "tests", "suite") - suite_files = FileUtils.load_folder_files(suite_def_folder) + def load_test_dependencies(): + """ load all api and suite definitions. + default api folder is "$CWD/tests/api/". + default suite folder is "$CWD/tests/suite/". + """ + # TODO: cache api and suite loading + # load api definitions + api_def_folder = os.path.join(os.getcwd(), "tests", "api") + for test_file in FileUtils.load_folder_files(api_def_folder): + TestcaseLoader.load_api_file(test_file) - for suite_file in suite_files: - suite = load_test_file(suite_file) - if "def" not in suite["config"]: - raise exception.ParamsError("def missed in suite file: {}!".format(suite_file)) + # load suite definitions + suite_def_folder = os.path.join(os.getcwd(), "tests", "suite") + for suite_file in FileUtils.load_folder_files(suite_def_folder): + suite = TestcaseLoader.load_test_file(suite_file) + if "def" not in suite["config"]: + raise exception.ParamsError("def missed in suite file: {}!".format(suite_file)) - call_func = suite["config"]["def"] - function_meta = parse_function(call_func) - suite["function_meta"] = function_meta - test_def_overall_dict["suite"][function_meta["func_name"]] = suite + call_func = suite["config"]["def"] + function_meta = parse_function(call_func) + suite["function_meta"] = function_meta + TestcaseLoader.overall_def_dict["suite"][function_meta["func_name"]] = suite -def load_testsets_by_path(path): - """ load testcases from file path - @param path: path could be in several type - - absolute/relative file path - - absolute/relative folder path - - list/set container with file(s) and/or folder(s) - @return testcase sets list, each testset is corresponding to a file - [ - testset_dict_1, - testset_dict_2 - ] - """ - if isinstance(path, (list, set)): - testsets = [] + def load_api_file(file_path): + """ load api definition from file and store in overall_def_dict["api"] + api file should be in format below: + [ + { + "api": { + "def": "api_login", + "request": {}, + "validate": [] + } + }, + { + "api": { + "def": "api_logout", + "request": {}, + "validate": [] + } + } + ] + """ + api_items = FileUtils.load_file(file_path) + if not isinstance(api_items, list): + raise exception.FileFormatError("API format error: {}".format(file_path)) - for file_path in set(path): - testset = load_testsets_by_path(file_path) - if not testset: - continue - testsets.extend(testset) + for api_item in api_items: + if not isinstance(api_item, dict) or len(api_item) != 1: + raise exception.FileFormatError("API format error: {}".format(file_path)) - return testsets + key, api_dict = api_item.popitem() + if key != "api" or not isinstance(api_dict, dict) or "def" not in api_dict: + raise exception.FileFormatError("API format error: {}".format(file_path)) - if not os.path.isabs(path): - path = os.path.join(os.getcwd(), path) + api_def = api_dict.pop("def") + function_meta = parse_function(api_def) + func_name = function_meta["func_name"] - if path in testcases_cache_mapping: - return testcases_cache_mapping[path] + if func_name in TestcaseLoader.overall_def_dict["api"]: + logger.log_warning("API definition duplicated: {}".format(func_name)) - if os.path.isdir(path): - files_list = FileUtils.load_folder_files(path) - testcases_list = load_testsets_by_path(files_list) + api_dict["function_meta"] = function_meta + TestcaseLoader.overall_def_dict["api"][func_name] = api_dict + + def load_test_file(file_path): + """ load testcase file or suite file + @param file_path: absolute valid file path + file_path should be in format below: + [ + { + "config": { + "name": "", + "def": "suite_order()", + "request": {} + } + }, + { + "test": { + "name": "add product to cart", + "api": "api_add_cart()", + "validate": [] + } + }, + { + "test": { + "name": "checkout cart", + "request": {}, + "validate": [] + } + } + ] + @return testset dict + { + "name": "desc1", + "config": {}, + "testcases": [testcase11, testcase12] + } + """ + testset = { + "name": "", + "config": { + "path": file_path + }, + "testcases": [] # TODO: rename to tests + } + for item in FileUtils.load_file(file_path): + if not isinstance(item, dict) or len(item) != 1: + raise exception.FileFormatError("Testcase format error: {}".format(file_path)) + + key, test_block = item.popitem() + if not isinstance(test_block, dict): + raise exception.FileFormatError("Testcase format error: {}".format(file_path)) + + if key == "config": + testset["config"].update(test_block) + testset["name"] = test_block.get("name", "") + + elif key == "test": + if "api" in test_block: + ref_call = test_block["api"] + def_block = TestcaseLoader._get_block_by_name(ref_call, "api") + TestcaseLoader._override_block(def_block, test_block) + testset["testcases"].append(test_block) + elif "suite" in test_block: + ref_call = test_block["suite"] + block = TestcaseLoader._get_block_by_name(ref_call, "suite") + testset["testcases"].extend(block["testcases"]) + else: + testset["testcases"].append(test_block) - elif os.path.isfile(path): - try: - testset = load_test_file(path) - if testset["testcases"] or testset["api"]: - testcases_list = [testset] else: + logger.log_warning( + "unexpected block key: {}. block key should only be 'config' or 'test'.".format(key) + ) + + return testset + + def _get_block_by_name(ref_call, ref_type): + """ get test content by reference name + @params: + ref_call: e.g. api_v1_Account_Login_POST($UserName, $Password) + ref_type: "api" or "suite" + """ + function_meta = parse_function(ref_call) + func_name = function_meta["func_name"] + call_args = function_meta["args"] + block = TestcaseLoader._get_test_definition(func_name, ref_type) + def_args = block.get("function_meta").get("args", []) + + if len(call_args) != len(def_args): + raise exception.ParamsError("call args mismatch defined args!") + + args_mapping = {} + for index, item in enumerate(def_args): + if call_args[index] == item: + continue + + args_mapping[item] = call_args[index] + + if args_mapping: + block = substitute_variables_with_mapping(block, args_mapping) + + return block + + def _get_test_definition(name, ref_type): + """ get expected api or suite. + @params: + name: api or suite name + ref_type: "api" or "suite" + @return + expected api info if found, otherwise raise ApiNotFound exception + """ + block = TestcaseLoader.overall_def_dict.get(ref_type, {}).get(name) + + if not block: + err_msg = "{} not found!".format(name) + if ref_type == "api": + raise exception.ApiNotFound(err_msg) + else: + # ref_type == "suite": + raise exception.SuiteNotFound(err_msg) + + return block + + def _override_block(def_block, current_block): + """ override def_block with current_block + @param def_block: + { + "name": "get token", + "request": {...}, + "validate": [{'eq': ['status_code', 200]}] + } + @param current_block: + { + "name": "get token", + "extract": [{"token": "content.token"}], + "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] + } + @return + { + "name": "get token", + "request": {...}, + "extract": [{"token": "content.token"}], + "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}] + } + """ + def_validators = def_block.get("validate") or def_block.get("validators", []) + current_validators = current_block.get("validate") or current_block.get("validators", []) + + def_extrators = def_block.get("extract") \ + or def_block.get("extractors") \ + or def_block.get("extract_binds", []) + current_extractors = current_block.get("extract") \ + or current_block.get("extractors") \ + or current_block.get("extract_binds", []) + + current_block.update(def_block) + current_block["validate"] = _merge_validator( + def_validators, + current_validators + ) + current_block["extract"] = _merge_extractor( + def_extrators, + current_extractors + ) + + def load_testsets_by_path(path): + """ load testcases from file path + @param path: path could be in several type + - absolute/relative file path + - absolute/relative folder path + - list/set container with file(s) and/or folder(s) + @return testcase sets list, each testset is corresponding to a file + [ + testset_dict_1, + testset_dict_2 + ] + """ + if isinstance(path, (list, set)): + testsets = [] + + for file_path in set(path): + testset = TestcaseLoader.load_testsets_by_path(file_path) + if not testset: + continue + testsets.extend(testset) + + return testsets + + if not os.path.isabs(path): + path = os.path.join(os.getcwd(), path) + + if path in TestcaseLoader.testcases_cache_mapping: + return TestcaseLoader.testcases_cache_mapping[path] + + if os.path.isdir(path): + files_list = FileUtils.load_folder_files(path) + testcases_list = TestcaseLoader.load_testsets_by_path(files_list) + + elif os.path.isfile(path): + try: + testset = TestcaseLoader.load_test_file(path) + if testset["testcases"] or testset["api"]: + testcases_list = [testset] + else: + testcases_list = [] + except exception.FileFormatError: testcases_list = [] - except exception.FileFormatError: + + else: + logger.log_error(u"file not found: {}".format(path)) testcases_list = [] - else: - logger.log_error(u"file not found: {}".format(path)) - testcases_list = [] - - testcases_cache_mapping[path] = testcases_list - return testcases_list + TestcaseLoader.testcases_cache_mapping[path] = testcases_list + return testcases_list def parse_validator(validator): """ parse validator, validator maybe in two format @@ -262,11 +463,11 @@ def _get_validators_mapping(validators): return validators_mapping -def merge_validator(api_validators, test_validators): - """ merge api_validators with test_validators +def _merge_validator(def_validators, current_validators): + """ merge def_validators with current_validators @params: - api_validators: [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] - test_validators: [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] + def_validators: [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}] + current_validators: [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}] @return: [ {"check": "v1", "expect": 201, "comparator": "eq"}, @@ -274,24 +475,24 @@ def merge_validator(api_validators, test_validators): {"check": "s3", "expect": 12, "comparator": "len_eq"} ] """ - if not api_validators: - return test_validators + if not def_validators: + return current_validators - elif not test_validators: - return api_validators + elif not current_validators: + return def_validators else: - api_validators_mapping = _get_validators_mapping(api_validators) - test_validators_mapping = _get_validators_mapping(test_validators) + api_validators_mapping = _get_validators_mapping(def_validators) + test_validators_mapping = _get_validators_mapping(current_validators) api_validators_mapping.update(test_validators_mapping) return list(api_validators_mapping.values()) -def merge_extractor(api_extrators, test_extracors): - """ merge api_extrators with test_extracors +def _merge_extractor(def_extrators, current_extractors): + """ merge def_extrators with current_extractors @params: - api_extrators: [{"var1": "val1"}, {"var2": "val2"}] - test_extracors: [{"var1": "val111"}, {"var3": "val3"}] + def_extrators: [{"var1": "val1"}, {"var2": "val2"}] + current_extractors: [{"var1": "val111"}, {"var3": "val3"}] @return: [ {"var1": "val111"}, @@ -299,15 +500,15 @@ def merge_extractor(api_extrators, test_extracors): {"var3": "val3"} ] """ - if not api_extrators: - return test_extracors + if not def_extrators: + return current_extractors - elif not test_extracors: - return api_extrators + elif not current_extractors: + return def_extrators else: extractor_dict = OrderedDict() - for api_extrator in api_extrators: + for api_extrator in def_extrators: if len(api_extrator) != 1: logger.log_warning("incorrect extractor: {}".format(api_extrator)) continue @@ -315,7 +516,7 @@ def merge_extractor(api_extrators, test_extracors): var_name = list(api_extrator.keys())[0] extractor_dict[var_name] = api_extrator[var_name] - for test_extrator in test_extracors: + for test_extrator in current_extractors: if len(test_extrator) != 1: logger.log_warning("incorrect extractor: {}".format(test_extrator)) continue @@ -329,46 +530,6 @@ def merge_extractor(api_extrators, test_extracors): return extractor_list -def extend_test_api(test_block_dict): - """ update test block api with api definition - @param - test_block_dict: - { - "name": "get token", - "api": "get_token($user_agent, $device_sn, $os_platform, $app_version)", - "extract": [{"token": "content.token"}], - "validate": [{'eq': ['status_code', 200]}, {'len_eq': ['content.token', 16]}] - } - @return - { - "name": "get token", - "request": {...}, - "extract": [{"token": "content.token"}], - "validate": [{'eq': ['status_code', 200]}, {'len_eq': ['content.token', 16]}] - } - """ - ref_name = test_block_dict["api"] - test_info = get_testinfo_by_reference(ref_name, "api") - - api_validators = test_info.get("validate") or test_info.get("validators", []) - test_validators = test_block_dict.get("validate") or test_block_dict.get("validators", []) - - api_extrators = test_info.get("extract") \ - or test_info.get("extractors") \ - or test_info.get("extract_binds", []) - test_extracors = test_block_dict.get("extract") \ - or test_block_dict.get("extractors") \ - or test_block_dict.get("extract_binds", []) - - test_block_dict.update(test_info) - test_block_dict["validate"] = merge_validator( - api_validators, - test_validators - ) - test_block_dict["extract"] = merge_extractor( - api_extrators, - test_extracors - ) def is_testset(data_structure): """ check if data_structure is a testset @@ -410,115 +571,6 @@ def is_testsets(data_structure): return True -def load_test_file(file_path): - """ load testset file, get testset data structure. - @param file_path: absolute valid testset file path - @return testset dict - { - "name": "desc1", - "config": {}, - "api": {}, - "testcases": [testcase11, testcase12] - } - """ - testset = { - "name": "", - "config": { - "path": file_path - }, - "api": {}, - "testcases": [] - } - tests_list = FileUtils.load_file(file_path) - - for item in tests_list: - for key in item: - if key == "config": - testset["config"].update(item["config"]) - testset["name"] = item["config"].get("name", "") - - elif key == "test": - test_block_dict = item["test"] - if "api" in test_block_dict: - extend_test_api(test_block_dict) - testset["testcases"].append(test_block_dict) - elif "suite" in test_block_dict: - ref_name = test_block_dict["suite"] - test_info = get_testinfo_by_reference(ref_name, "suite") - testset["testcases"].extend(test_info["testcases"]) - else: - testset["testcases"].append(test_block_dict) - - elif key == "api": - api_def = item["api"].pop("def") - function_meta = parse_function(api_def) - func_name = function_meta["func_name"] - - if func_name in testset["api"]: - logger.log_warning("api definition duplicated: {}".format(func_name)) - - api_info = {} - api_info["function_meta"] = function_meta - api_info.update(item["api"]) - testset["api"][func_name] = api_info - - else: - logger.log_warning( - "unexpected block: {}. block should only be 'config', 'test' or 'api'.".format(key) - ) - - return testset - -def get_testinfo_by_reference(ref_name, ref_type): - """ get test content by reference name - @params: - ref_name: reference name, e.g. api_v1_Account_Login_POST($UserName, $Password) - ref_type: "api" or "suite" - """ - function_meta = parse_function(ref_name) - func_name = function_meta["func_name"] - call_args = function_meta["args"] - test_info = get_test_definition(func_name, ref_type) - def_args = test_info.get("function_meta").get("args", []) - - if len(call_args) != len(def_args): - raise exception.ParamsError("call args mismatch defined args!") - - args_mapping = {} - for index, item in enumerate(def_args): - if call_args[index] == item: - continue - - args_mapping[item] = call_args[index] - - if args_mapping: - test_info = substitute_variables_with_mapping(test_info, args_mapping) - - return test_info - -def get_test_definition(name, ref_type): - """ get expected api or suite. - @params: - name: api or suite name - ref_type: "api" or "suite" - @return - expected api info if found, otherwise raise ApiNotFound exception - """ - if not test_def_overall_dict.get("loaded", False): - load_test_dependencies() - - test_info = test_def_overall_dict.get(ref_type, {}).get(name) - if not test_info: - err_msg = "{} {} not found!".format(ref_type, name) - if ref_type == "api": - raise exception.ApiNotFound(err_msg) - elif ref_type == "suite": - raise exception.SuiteNotFound(err_msg) - else: - raise exception.ParamsError("ref_type can only be api or suite!") - - return test_info - def substitute_variables_with_mapping(content, mapping): """ substitute variables in content with mapping e.g. diff --git a/examples/HelloWorld/tests/api/basic.yml b/tests/api/basic.yml similarity index 93% rename from examples/HelloWorld/tests/api/basic.yml rename to tests/api/basic.yml index b56925f6..a135bac8 100644 --- a/examples/HelloWorld/tests/api/basic.yml +++ b/tests/api/basic.yml @@ -11,8 +11,9 @@ json: sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)} validate: - - eq: ["status_code", 200] - - len_eq: ["content.token", 16] + - eq: ["status_code", 0] + - len_eq: ["content.token", 12] + - contains: [{"a": 1, "b": 2}, "a"] - api: def: create_user($uid, $user_name, $user_password, $token) diff --git a/tests/api/demo.yml b/tests/api/demo.yml deleted file mode 100644 index ee95575e..00000000 --- a/tests/api/demo.yml +++ /dev/null @@ -1,70 +0,0 @@ -- api: - def: get_token($user_agent, $device_sn, $os_platform, $app_version) - request: - url: /api/get-token - method: POST - headers: - user_agent: $user_agent - device_sn: $device_sn - os_platform: $os_platform - app_version: $app_version - json: - sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)} - validate: - - "eq": ["status_code", 0] - - "len_eq": ["content.token", 12] - - "contains": [{"a": 1, "b": 2}, "a"] - -- api: - def: create_user($uid, $user_name, $user_password, $token) - request: - url: /api/users/$uid - method: POST - headers: - token: $token - json: - name: $user_name - password: $user_password - -- api: - def: get_user($uid, $token) - request: - url: /api/users/$uid - method: GET - headers: - token: $token - -- api: - def: update_user($uid, $user_name, $user_password, $token) - request: - url: /api/users/$uid - method: PUT - headers: - token: $token - json: - name: $user_name - password: $user_password - -- api: - def: delete_user($uid, $token) - request: - url: /api/users/$uid - method: DELETE - headers: - token: $token - -- api: - def: get_users($token) - request: - url: /api/users - method: GET - headers: - token: $token - -- api: - def: reset_all($token) - request: - url: /api/reset-all - method: GET - headers: - token: $token diff --git a/examples/HelloWorld/tests/suite/create_and_check.yml b/tests/suite/create_and_get.yml similarity index 100% rename from examples/HelloWorld/tests/suite/create_and_check.yml rename to tests/suite/create_and_get.yml diff --git a/examples/HelloWorld/tests/suite/setup.yml b/tests/suite/setup.yml similarity index 100% rename from examples/HelloWorld/tests/suite/setup.yml rename to tests/suite/setup.yml diff --git a/tests/test_runner.py b/tests/test_runner.py index 49a2f449..d8c4f7ac 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,7 +1,8 @@ import os import time -from httprunner import HttpRunner, exception, runner, testcase +from httprunner import HttpRunner, exception, runner +from httprunner.testcase import TestcaseLoader from httprunner.utils import FileUtils, deep_update_dict from tests.base import ApiServerUnittest @@ -154,7 +155,7 @@ class TestRunner(ApiServerUnittest): def test_run_testcase_with_empty_header(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/test_bugfix.yml') - testsets = testcase.load_testsets_by_path(testcase_file_path) + testsets = TestcaseLoader.load_testsets_by_path(testcase_file_path) testset = testsets[0] config_dict_headers = testset["config"]["request"]["headers"] test_dict_headers = testset["testcases"][0]["request"]["headers"] diff --git a/tests/test_task.py b/tests/test_task.py index 8fa3614f..22000023 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -1,7 +1,7 @@ import os from httprunner import task -from httprunner.testcase import load_test_file +from httprunner.testcase import TestcaseLoader from tests.base import ApiServerUnittest @@ -17,7 +17,7 @@ class TestTask(ApiServerUnittest): def test_create_suite(self): testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_testset_variables.yml') - testset = load_test_file(testcase_file_path) + testset = TestcaseLoader.load_test_file(testcase_file_path) suite = task.TestSuite(testset) self.assertEqual(suite.countTestCases(), 3) for testcase in suite: diff --git a/tests/test_testcase.py b/tests/test_testcase.py index 8a0a0e76..00ff32f4 100644 --- a/tests/test_testcase.py +++ b/tests/test_testcase.py @@ -4,7 +4,192 @@ import unittest from httprunner import testcase from httprunner.exception import (ApiNotFound, FileFormatError, - FileNotFoundError, ParamsError) + FileNotFoundError, ParamsError, + SuiteNotFound) +from httprunner.testcase import TestcaseLoader + + +class TestTestcaseLoader(unittest.TestCase): + + def setUp(self): + TestcaseLoader.overall_def_dict = { + "api": {}, + "suite": {} + } + + def test_load_test_dependencies(self): + TestcaseLoader.load_test_dependencies() + overall_def_dict = TestcaseLoader.overall_def_dict + self.assertIn("get_token", overall_def_dict["api"]) + self.assertIn("create_and_check", overall_def_dict["suite"]) + + def test_load_api_file(self): + TestcaseLoader.load_api_file("tests/api/basic.yml") + overall_api_def_dict = TestcaseLoader.overall_def_dict["api"] + self.assertIn("get_token",overall_api_def_dict) + self.assertEqual("/api/get-token", overall_api_def_dict["get_token"]["request"]["url"]) + self.assertIn("$user_agent", overall_api_def_dict["get_token"]["function_meta"]["args"]) + self.assertEqual(len(overall_api_def_dict["get_token"]["validate"]), 3) + + def test_load_test_file_suite(self): + TestcaseLoader.load_api_file("tests/api/basic.yml") + testset = TestcaseLoader.load_test_file("tests/suite/create_and_get.yml") + self.assertEqual(testset["name"], "create user and check result.") + self.assertEqual(testset["config"]["name"], "create user and check result.") + self.assertEqual(len(testset["testcases"]), 3) + self.assertEqual(testset["testcases"][0]["name"], "make sure user $uid does not exist") + self.assertEqual(testset["testcases"][0]["request"]["url"], "/api/users/$uid") + + def test_load_test_file_testcase(self): + TestcaseLoader.load_test_dependencies() + testset = TestcaseLoader.load_test_file("tests/testcases/smoketest.yml") + self.assertEqual(testset["name"], "smoketest") + self.assertEqual(testset["config"]["path"], "tests/testcases/smoketest.yml") + self.assertIn("device_sn", testset["config"]["variables"][0]) + self.assertEqual(len(testset["testcases"]), 8) + self.assertEqual(testset["testcases"][0]["name"], "get token") + + def test_get_block_by_name(self): + TestcaseLoader.load_test_dependencies() + ref_call = "get_user($uid, $token)" + block = TestcaseLoader._get_block_by_name(ref_call, "api") + self.assertEqual(block["request"]["url"], "/api/users/$uid") + self.assertEqual(block["function_meta"]["func_name"], "get_user") + self.assertEqual(block["function_meta"]["args"], ['$uid', '$token']) + + def test_get_block_by_name_args_mismatch(self): + TestcaseLoader.load_test_dependencies() + ref_call = "get_user($uid, $token, $var)" + with self.assertRaises(ParamsError): + TestcaseLoader._get_block_by_name(ref_call, "api") + + def test_get_test_definition_api(self): + TestcaseLoader.load_test_dependencies() + api_def = TestcaseLoader._get_test_definition("get_token", "api") + self.assertEqual(api_def["request"]["url"], "/api/get-token") + + with self.assertRaises(ApiNotFound): + TestcaseLoader._get_test_definition("get_token_XXX", "api") + + def test_get_test_definition_suite(self): + TestcaseLoader.load_test_dependencies() + api_def = TestcaseLoader._get_test_definition("create_and_check", "suite") + self.assertEqual(api_def["name"], "create user and check result.") + + with self.assertRaises(SuiteNotFound): + TestcaseLoader._get_test_definition("create_and_check_XXX", "suite") + + def test_override_block(self): + TestcaseLoader.load_test_dependencies() + def_block = TestcaseLoader._get_block_by_name("get_token($user_agent, $device_sn, $os_platform, $app_version)", "api") + test_block = { + "name": "override block", + "variables": [ + {"var": 123} + ], + 'request': { + 'url': '/api/get-token', 'method': 'POST', 'headers': {'user_agent': '$user_agent', 'device_sn': '$device_sn', 'os_platform': '$os_platform', 'app_version': '$app_version'}, 'json': {'sign': '${get_sign($user_agent, $device_sn, $os_platform, $app_version)}'}}, + 'validate': [ + {'eq': ['status_code', 201]}, + {'len_eq': ['content.token', 32]} + ] + } + + TestcaseLoader._override_block(def_block, test_block) + self.assertEqual(test_block["name"], "override block") + self.assertEqual(test_block["validate"][0], {'check': 'status_code', 'expect': 201, 'comparator': 'eq'}) + self.assertEqual(test_block["validate"][1], {'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}) + + def test_load_testcases_by_path_files(self): + testsets_list = [] + + # absolute file path + path = os.path.join( + os.getcwd(), 'tests/data/demo_testset_hardcode.json') + testset_list = TestcaseLoader.load_testsets_by_path(path) + self.assertEqual(len(testset_list), 1) + self.assertIn("path", testset_list[0]["config"]) + self.assertEqual(testset_list[0]["config"]["path"], path) + self.assertEqual(len(testset_list[0]["testcases"]), 3) + testsets_list.extend(testset_list) + + # relative file path + path = 'tests/data/demo_testset_hardcode.yml' + testset_list = TestcaseLoader.load_testsets_by_path(path) + self.assertEqual(len(testset_list), 1) + self.assertIn("path", testset_list[0]["config"]) + self.assertIn(path, testset_list[0]["config"]["path"]) + self.assertEqual(len(testset_list[0]["testcases"]), 3) + testsets_list.extend(testset_list) + + # list/set container with file(s) + path = [ + os.path.join(os.getcwd(), 'tests/data/demo_testset_hardcode.json'), + 'tests/data/demo_testset_hardcode.yml' + ] + testset_list = TestcaseLoader.load_testsets_by_path(path) + self.assertEqual(len(testset_list), 2) + self.assertEqual(len(testset_list[0]["testcases"]), 3) + self.assertEqual(len(testset_list[1]["testcases"]), 3) + testsets_list.extend(testset_list) + self.assertEqual(len(testsets_list), 4) + + for testset in testsets_list: + for test in testset["testcases"]: + self.assertIn('name', test) + self.assertIn('request', test) + self.assertIn('url', test['request']) + self.assertIn('method', test['request']) + + def test_load_testcases_by_path_folder(self): + TestcaseLoader.load_test_dependencies() + # absolute folder path + path = os.path.join(os.getcwd(), 'tests/data') + testset_list_1 = TestcaseLoader.load_testsets_by_path(path) + self.assertGreater(len(testset_list_1), 4) + + # relative folder path + path = 'tests/data/' + testset_list_2 = TestcaseLoader.load_testsets_by_path(path) + self.assertEqual(len(testset_list_1), len(testset_list_2)) + + # list/set container with file(s) + path = [ + os.path.join(os.getcwd(), 'tests/data'), + 'tests/data/' + ] + testset_list_3 = TestcaseLoader.load_testsets_by_path(path) + self.assertEqual(len(testset_list_3), 2 * len(testset_list_1)) + + def test_load_testcases_by_path_not_exist(self): + # absolute folder path + path = os.path.join(os.getcwd(), 'tests/data_not_exist') + testset_list_1 = TestcaseLoader.load_testsets_by_path(path) + self.assertEqual(testset_list_1, []) + + # relative folder path + path = 'tests/data_not_exist' + testset_list_2 = TestcaseLoader.load_testsets_by_path(path) + self.assertEqual(testset_list_2, []) + + # list/set container with file(s) + path = [ + os.path.join(os.getcwd(), 'tests/data_not_exist'), + 'tests/data_not_exist/' + ] + testset_list_3 = TestcaseLoader.load_testsets_by_path(path) + self.assertEqual(testset_list_3, []) + + def test_load_testcases_by_path_layered(self): + TestcaseLoader.load_test_dependencies() + path = os.path.join( + os.getcwd(), 'tests/data/demo_testset_layer.yml') + testsets_list = TestcaseLoader.load_testsets_by_path(path) + self.assertIn("variables", testsets_list[0]["config"]) + self.assertIn("request", testsets_list[0]["config"]) + self.assertIn("request", testsets_list[0]["testcases"][0]) + self.assertIn("url", testsets_list[0]["testcases"][0]["request"]) + self.assertIn("validate", testsets_list[0]["testcases"][0]) class TestcaseParserUnittest(unittest.TestCase): @@ -448,94 +633,6 @@ class TestcaseParserUnittest(unittest.TestCase): 3 ) - def test_load_testcases_by_path_files(self): - testsets_list = [] - - # absolute file path - path = os.path.join( - os.getcwd(), 'tests/data/demo_testset_hardcode.json') - testset_list = testcase.load_testsets_by_path(path) - self.assertEqual(len(testset_list), 1) - self.assertIn("path", testset_list[0]["config"]) - self.assertEqual(testset_list[0]["config"]["path"], path) - self.assertEqual(len(testset_list[0]["testcases"]), 3) - testsets_list.extend(testset_list) - - # relative file path - path = 'tests/data/demo_testset_hardcode.yml' - testset_list = testcase.load_testsets_by_path(path) - self.assertEqual(len(testset_list), 1) - self.assertIn("path", testset_list[0]["config"]) - self.assertIn(path, testset_list[0]["config"]["path"]) - self.assertEqual(len(testset_list[0]["testcases"]), 3) - testsets_list.extend(testset_list) - - # list/set container with file(s) - path = [ - os.path.join(os.getcwd(), 'tests/data/demo_testset_hardcode.json'), - 'tests/data/demo_testset_hardcode.yml' - ] - testset_list = testcase.load_testsets_by_path(path) - self.assertEqual(len(testset_list), 2) - self.assertEqual(len(testset_list[0]["testcases"]), 3) - self.assertEqual(len(testset_list[1]["testcases"]), 3) - testsets_list.extend(testset_list) - self.assertEqual(len(testsets_list), 4) - - for testset in testsets_list: - for test in testset["testcases"]: - self.assertIn('name', test) - self.assertIn('request', test) - self.assertIn('url', test['request']) - self.assertIn('method', test['request']) - - def test_load_testcases_by_path_folder(self): - # absolute folder path - path = os.path.join(os.getcwd(), 'tests/data') - testset_list_1 = testcase.load_testsets_by_path(path) - self.assertGreater(len(testset_list_1), 4) - - # relative folder path - path = 'tests/data/' - testset_list_2 = testcase.load_testsets_by_path(path) - self.assertEqual(len(testset_list_1), len(testset_list_2)) - - # list/set container with file(s) - path = [ - os.path.join(os.getcwd(), 'tests/data'), - 'tests/data/' - ] - testset_list_3 = testcase.load_testsets_by_path(path) - self.assertEqual(len(testset_list_3), 2 * len(testset_list_1)) - - def test_load_testcases_by_path_not_exist(self): - # absolute folder path - path = os.path.join(os.getcwd(), 'tests/data_not_exist') - testset_list_1 = testcase.load_testsets_by_path(path) - self.assertEqual(testset_list_1, []) - - # relative folder path - path = 'tests/data_not_exist' - testset_list_2 = testcase.load_testsets_by_path(path) - self.assertEqual(testset_list_2, []) - - # list/set container with file(s) - path = [ - os.path.join(os.getcwd(), 'tests/data_not_exist'), - 'tests/data_not_exist/' - ] - testset_list_3 = testcase.load_testsets_by_path(path) - self.assertEqual(testset_list_3, []) - - def test_load_testcases_by_path_layered(self): - path = os.path.join( - os.getcwd(), 'tests/data/demo_testset_layer.yml') - testsets_list = testcase.load_testsets_by_path(path) - self.assertIn("variables", testsets_list[0]["config"]) - self.assertIn("request", testsets_list[0]["config"]) - self.assertIn("request", testsets_list[0]["testcases"][0]) - self.assertIn("url", testsets_list[0]["testcases"][0]["request"]) - self.assertIn("validate", testsets_list[0]["testcases"][0]) def test_substitute_variables_with_mapping(self): content = { @@ -564,23 +661,6 @@ class TestcaseParserUnittest(unittest.TestCase): self.assertFalse(result["request"]["data"]["false"]) self.assertEqual("", result["request"]["data"]["empty_str"]) - def test_load_test_dependencies(self): - testcase.test_def_overall_dict = {} - testcase.load_test_dependencies() - self.assertTrue(testcase.test_def_overall_dict["loaded"]) - api_dict = testcase.test_def_overall_dict["api"] - self.assertIn("get_token", api_dict) - self.assertEqual("/api/get-token", api_dict["get_token"]["request"]["url"]) - self.assertIn("$user_agent", api_dict["get_token"]["function_meta"]["args"]) - self.assertIn("create_user", api_dict) - - def test_get_api_definition(self): - api_info = testcase.get_test_definition("get_token", "api") - self.assertEqual("/api/get-token", api_info["request"]["url"]) - self.assertIn("get_token", testcase.test_def_overall_dict["api"]) - - with self.assertRaises(ApiNotFound): - testcase.get_test_definition("api_not_exist", "api") def test_parse_validator(self): validator = {"check": "status_code", "comparator": "eq", "expect": 201} @@ -596,16 +676,16 @@ class TestcaseParserUnittest(unittest.TestCase): ) def test_merge_validator(self): - api_validators = [ + def_validators = [ {'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"} ] - test_validators = [ + current_validators = [ {"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]} ] - merged_validators = testcase.merge_validator(api_validators, test_validators) + merged_validators = testcase._merge_validator(def_validators, current_validators) self.assertIn( {"check": "v1", "expect": 201, "comparator": "eq"}, merged_validators @@ -620,25 +700,25 @@ class TestcaseParserUnittest(unittest.TestCase): ) def test_merge_validator_with_dict(self): - api_validators = [ + def_validators = [ {'eq': ["a", {"v": 1}]}, {'eq': [{"b": 1}, 200]} ] - test_validators = [ + current_validators = [ {'len_eq': ['s3', 12]}, {'eq': [{"b": 1}, 201]} ] - merged_validators = testcase.merge_validator(api_validators, test_validators) + merged_validators = testcase._merge_validator(def_validators, current_validators) self.assertEqual(len(merged_validators), 3) self.assertIn({'check': {'b': 1}, 'expect': 201, 'comparator': 'eq'}, merged_validators) self.assertNotIn({'check': {'b': 1}, 'expect': 200, 'comparator': 'eq'}, merged_validators) def test_merge_extractor(self): api_extrators = [{"var1": "val1"}, {"var2": "val2"}] - test_extracors = [{"var1": "val111"}, {"var3": "val3"}] + current_extractors = [{"var1": "val111"}, {"var3": "val3"}] - merged_extractors = testcase.merge_extractor(api_extrators, test_extracors) + merged_extractors = testcase._merge_extractor(api_extrators, current_extractors) self.assertIn( {"var1": "val111"}, merged_extractors diff --git a/tests/test_utils.py b/tests/test_utils.py index a26dc2fd..175dea28 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -126,11 +126,11 @@ class TestFileUtils(unittest.TestCase): self.assertNotIn(file1, files) files_1 = FileUtils.load_folder_files(folder) - api_file = os.path.join(os.getcwd(), 'tests', 'api', 'demo.yml') + api_file = os.path.join(os.getcwd(), 'tests', 'api', 'basic.yml') self.assertEqual(files_1[0], api_file) files_2 = FileUtils.load_folder_files(folder) - api_file = os.path.join(os.getcwd(), 'tests', 'api', 'demo.yml') + api_file = os.path.join(os.getcwd(), 'tests', 'api', 'basic.yml') self.assertEqual(files_2[0], api_file) self.assertEqual(len(files_1), len(files_2)) @@ -231,7 +231,7 @@ class TestUtils(ApiServerUnittest): self.assertEqual(utils.get_uniform_comparator("count_le"), "length_less_than_or_equals") self.assertEqual(utils.get_uniform_comparator("count_less_than_or_equals"), "length_less_than_or_equals") - def test_validators(self): + def current_validators(self): imported_module = utils.get_imported_module("httprunner.built_in") functions_mapping = utils.filter_module(imported_module, "function") diff --git a/examples/HelloWorld/tests/testcases/smoketest.yml b/tests/testcases/smoketest.yml similarity index 100% rename from examples/HelloWorld/tests/testcases/smoketest.yml rename to tests/testcases/smoketest.yml