From 9b0d708367d7af1845ea5a02227c631e6b135dd6 Mon Sep 17 00:00:00 2001
From: debugtalk <mail@debugtalk.com>
Date: Sun, 12 Aug 2018 01:25:28 +0800
Subject: [PATCH] refactor loader: load testcases

---
 httprunner/loader.py     | 505 +++++++++++++++++++--------------------
 httprunner/locusts.py    |   2 +-
 httprunner/task.py       |  61 +++--
 httprunner/validator.py  |  49 ++--
 tests/test_httprunner.py |   4 +-
 tests/test_loader.py     |  98 +++-----
 tests/test_runner.py     |   8 +-
 tests/test_task.py       |   2 +-
 8 files changed, 355 insertions(+), 374 deletions(-)

diff --git a/httprunner/loader.py b/httprunner/loader.py
index a569a2c3..1b5417fc 100644
--- a/httprunner/loader.py
+++ b/httprunner/loader.py
@@ -13,19 +13,19 @@ from httprunner.compat import OrderedDict
 project_mapping = {
     "debugtalk": {},
     "env": {},
-    "tests": {
-        "api": {},
-        "testcases": {}
-    }
+    "def-api": {},
+    "def-testcase": {}
 }
-""" dict: save project loaded api/testcases, environments and debugtalk.py module.
+""" dict: save project loaded api/testcases definitions, environments and debugtalk.py module.
 """
 
+testcases_cache_mapping = {}
+
+
 ###############################################################################
 ##   file loader
 ###############################################################################
 
-
 def _check_format(file_path, content):
     """ check testcase format if valid
     """
@@ -367,84 +367,12 @@ def get_module_item(module_mapping, item_type, item_name):
 ##   testcase loader
 ###############################################################################
 
-overall_def_dict = {
-    "api": {},
-    "suite": {}
-}
-testcases_cache_mapping = {}
-
-
-def _load_test_dependencies():
-    """ load all api and suite definitions.
-        default api folder is "$CWD/tests/api/".
-        default suite folder is "$CWD/tests/suite/".
-    """
-    # TODO: cache api and suite loading
-    # load api definitions
-    api_def_folder = os.path.join(os.getcwd(), "tests", "api")
-    for test_file in load_folder_files(api_def_folder):
-        _load_api_file(test_file)
-
-    # load suite definitions
-    suite_def_folder = os.path.join(os.getcwd(), "tests", "suite")
-    for suite_file in load_folder_files(suite_def_folder):
-        suite = _load_test_file(suite_file)
-        if "def" not in suite["config"]:
-            raise exceptions.ParamsError("def missed in suite file: {}!".format(suite_file))
-
-        call_func = suite["config"]["def"]
-        function_meta = parser.parse_function(call_func)
-        suite["function_meta"] = function_meta
-        overall_def_dict["suite"][function_meta["func_name"]] = suite
-
-
-def _load_api_file(file_path):
-    """ load api definition from file and store in overall_def_dict["api"]
-        api file should be in format below:
-            [
-                {
-                    "api": {
-                        "def": "api_login",
-                        "request": {},
-                        "validate": []
-                    }
-                },
-                {
-                    "api": {
-                        "def": "api_logout",
-                        "request": {},
-                        "validate": []
-                    }
-                }
-            ]
-    """
-    api_items = load_file(file_path)
-    if not isinstance(api_items, list):
-        raise exceptions.FileFormatError("API format error: {}".format(file_path))
-
-    for api_item in api_items:
-        if not isinstance(api_item, dict) or len(api_item) != 1:
-            raise exceptions.FileFormatError("API format error: {}".format(file_path))
-
-        key, api_dict = api_item.popitem()
-        if key != "api" or not isinstance(api_dict, dict) or "def" not in api_dict:
-            raise exceptions.FileFormatError("API format error: {}".format(file_path))
-
-        api_def = api_dict.pop("def")
-        function_meta = parser.parse_function(api_def)
-        func_name = function_meta["func_name"]
-
-        if func_name in overall_def_dict["api"]:
-            logger.log_warning("API definition duplicated: {}".format(func_name))
-
-        api_dict["function_meta"] = function_meta
-        overall_def_dict["api"][func_name] = api_dict
-
-
 def _load_test_file(file_path):
     """ load testcase file or testsuite file
-    @param file_path: absolute valid file path
-        file_path should be in format below:
+
+    Args:
+        file_path (str): absolute valid file path. file_path should be in the following format:
+
             [
                 {
                     "config": {
@@ -460,6 +388,13 @@ def _load_test_file(file_path):
                         "validate": []
                     }
                 },
+                {
+                    "test": {
+                        "name": "add product to cart",
+                        "suite": "create_and_check()",
+                        "validate": []
+                    }
+                },
                 {
                     "test": {
                         "name": "checkout cart",
@@ -468,19 +403,24 @@ def _load_test_file(file_path):
                     }
                 }
             ]
-    @return testset dict
-        {
-            "config": {},
-            "testcases": [testcase11, testcase12]
-        }
+
+    Returns:
+        dict: testcase dict
+            {
+                "config": {},
+                "teststeps": [teststep11, teststep12]
+            }
+
     """
-    testset = {
+    testcase = {
         "config": {
             "path": file_path
         },
-        "testcases": []     # TODO: rename to tests
+        "teststeps": []
     }
+
     for item in load_file(file_path):
+        # TODO: add json schema validation
         if not isinstance(item, dict) or len(item) != 1:
             raise exceptions.FileFormatError("Testcase format error: {}".format(file_path))
 
@@ -489,43 +429,69 @@ def _load_test_file(file_path):
             raise exceptions.FileFormatError("Testcase format error: {}".format(file_path))
 
         if key == "config":
-            testset["config"].update(test_block)
+            testcase["config"].update(test_block)
 
         elif key == "test":
+
+            def extend_api_definition(block):
+                ref_call = block["api"]
+                def_block = _get_block_by_name(ref_call, "def-api")
+                _extend_block(block, def_block)
+
+            # reference api
             if "api" in test_block:
-                ref_call = test_block["api"]
-                def_block = _get_block_by_name(ref_call, "api")
-                _override_block(def_block, test_block)
-                testset["testcases"].append(test_block)
-            elif "suite" in test_block:
+                extend_api_definition(test_block)
+                testcase["teststeps"].append(test_block)
+
+            # reference testcase
+            elif "suite" in test_block: # TODO: replace suite with testcase
                 ref_call = test_block["suite"]
-                block = _get_block_by_name(ref_call, "suite")
-                testset["testcases"].extend(block["testcases"])
+                block = _get_block_by_name(ref_call, "def-testcase")
+                # TODO: bugfix lost block config variables
+                for teststep in block["teststeps"]:
+                    if "api" in teststep:
+                        extend_api_definition(teststep)
+                    testcase["teststeps"].append(teststep)
+
+            # define directly
             else:
-                testset["testcases"].append(test_block)
+                testcase["teststeps"].append(test_block)
 
         else:
             logger.log_warning(
                 "unexpected block key: {}. block key should only be 'config' or 'test'.".format(key)
             )
 
-    return testset
+    return testcase
 
 
 def _get_block_by_name(ref_call, ref_type):
-    """ get test content by reference name
-    @params:
-        ref_call: e.g. api_v1_Account_Login_POST($UserName, $Password)
-        ref_type: "api" or "suite"
+    """ get test content by reference name.
+
+    Args:
+        ref_call (str): call function.
+            e.g. api_v1_Account_Login_POST($UserName, $Password)
+        ref_type (enum): "def-api" or "def-testcase"
+
+    Returns:
+        dict: api/testcase definition.
+
+    Raises:
+        exceptions.ParamsError: call args number is not equal to defined args number.
+
     """
     function_meta = parser.parse_function(ref_call)
     func_name = function_meta["func_name"]
     call_args = function_meta["args"]
     block = _get_test_definition(func_name, ref_type)
-    def_args = block.get("function_meta").get("args", [])
+    def_args = block.get("function_meta", {}).get("args", [])
 
     if len(call_args) != len(def_args):
-        raise exceptions.ParamsError("call args mismatch defined args!")
+        err_msg = "{}: call args number is not equal to defined args number!\n".format(func_name)
+        err_msg += "defined args: {}\n".format(def_args)
+        err_msg += "reference args: {}".format(call_args)
+        logger.log_error(err_msg)
+        raise exceptions.ParamsError(err_msg)
 
     args_mapping = {}
     for index, item in enumerate(def_args):
@@ -542,80 +508,104 @@ def _get_block_by_name(ref_call, ref_type):
 
 def _get_test_definition(name, ref_type):
     """ get expected api or testcase.
-    @params:
-        name: api or testcase name
-        ref_type: "api" or "suite"
-    @return
-        expected api info if found, otherwise raise ApiNotFound exception
+
+    Args:
+        name (str): api or testcase name
+        ref_type (enum): "def-api" or "def-testcase"
+
+    Returns:
+        dict: expected api/testcase info if found.
+
+    Raises:
+        exceptions.ApiNotFound: api not found
+        exceptions.TestcaseNotFound: testcase not found
+
     """
-    block = overall_def_dict.get(ref_type, {}).get(name)
+    block = project_mapping.get(ref_type, {}).get(name)
 
     if not block:
         err_msg = "{} not found!".format(name)
-        if ref_type == "api":
+        if ref_type == "def-api":
             raise exceptions.ApiNotFound(err_msg)
         else:
-            # ref_type == "suite":
+            # ref_type == "def-testcase":
             raise exceptions.TestcaseNotFound(err_msg)
 
     return block
 
 
-def _override_block(def_block, current_block):
-    """ override def_block with current_block
-    @param def_block:
-        {
-            "name": "get token",
-            "request": {...},
-            "validate": [{'eq': ['status_code', 200]}]
-        }
-    @param current_block:
-        {
-            "name": "get token",
-            "extract": [{"token": "content.token"}],
-            "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]
-        }
-    @return
-        {
-            "name": "get token",
-            "request": {...},
-            "extract": [{"token": "content.token"}],
-            "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]
-        }
+def _extend_block(ref_block, def_block):
+    """ extend ref_block with def_block.
+
+    Args:
+        def_block (dict): api definition dict.
+        ref_block (dict): reference block
+
+    Returns:
+        dict: extended reference block.
+
+    Examples:
+        >>> def_block = {
+                "name": "get token 1",
+                "request": {...},
+                "validate": [{'eq': ['status_code', 200]}]
+            }
+        >>> ref_block = {
+                "name": "get token 2",
+                "extract": [{"token": "content.token"}],
+                "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]
+            }
+        >>> _extend_block(def_block, ref_block)
+            {
+                "name": "get token 2",
+                "request": {...},
+                "extract": [{"token": "content.token"}],
+                "validate": [{'eq': ['status_code', 201]}, {'len_eq': ['content.token', 16]}]
+            }
+
     """
+    # TODO: override variables
     def_validators = def_block.get("validate") or def_block.get("validators", [])
-    current_validators = current_block.get("validate") or current_block.get("validators", [])
+    ref_validators = ref_block.get("validate") or ref_block.get("validators", [])
 
     def_extrators = def_block.get("extract") \
         or def_block.get("extractors") \
         or def_block.get("extract_binds", [])
-    current_extractors = current_block.get("extract") \
-        or current_block.get("extractors") \
-        or current_block.get("extract_binds", [])
+    ref_extractors = ref_block.get("extract") \
+        or ref_block.get("extractors") \
+        or ref_block.get("extract_binds", [])
 
-    current_block.update(def_block)
-    current_block["validate"] = _merge_validator(
+    ref_block.update(def_block)
+    ref_block["validate"] = _merge_validator(
         def_validators,
-        current_validators
+        ref_validators
     )
-    current_block["extract"] = _merge_extractor(
+    ref_block["extract"] = _merge_extractor(
         def_extrators,
-        current_extractors
+        ref_extractors
     )
 
 
-def _get_validators_mapping(validators):
-    """ get validators mapping from api or test validators
-    @param (list) validators:
-        [
-            {"check": "v1", "expect": 201, "comparator": "eq"},
-            {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
-        ]
-    @return
-        {
-            ("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
-            ('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
-        }
+def _convert_validators_to_mapping(validators):
+    """ convert validators list to mapping.
+
+    Args:
+        validators (list): validators in list
+
+    Returns:
+        dict: validators mapping, use (check, comparator) as key.
+
+    Examples:
+        >>> validators = [
+                {"check": "v1", "expect": 201, "comparator": "eq"},
+                {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
+            ]
+        >>> _convert_validators_to_mapping(validators)
+            {
+                ("v1", "eq"): {"check": "v1", "expect": 201, "comparator": "eq"},
+                ('{"b": 1}', "eq"): {"check": {"b": 1}, "expect": 200, "comparator": "eq"}
+            }
+
     """
     validators_mapping = {}
 
@@ -633,48 +623,66 @@ def _get_validators_mapping(validators):
     return validators_mapping
 
 
-def _merge_validator(def_validators, current_validators):
-    """ merge def_validators with current_validators
-    @params:
-        def_validators: [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}]
-        current_validators: [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}]
-    @return:
-        [
-            {"check": "v1", "expect": 201, "comparator": "eq"},
-            {"check": "s2", "expect": 16, "comparator": "len_eq"},
-            {"check": "s3", "expect": 12, "comparator": "len_eq"}
-        ]
+def _merge_validator(def_validators, ref_validators):
+    """ merge def_validators with ref_validators.
+
+    Args:
+        def_validators (list):
+        ref_validators (list):
+
+    Returns:
+        list: merged validators
+
+    Examples:
+        >>> def_validators = [{'eq': ['v1', 200]}, {"check": "s2", "expect": 16, "comparator": "len_eq"}]
+        >>> ref_validators = [{"check": "v1", "expect": 201}, {'len_eq': ['s3', 12]}]
+        >>> _merge_validator(def_validators, ref_validators)
+            [
+                {"check": "v1", "expect": 201, "comparator": "eq"},
+                {"check": "s2", "expect": 16, "comparator": "len_eq"},
+                {"check": "s3", "expect": 12, "comparator": "len_eq"}
+            ]
+
     """
     if not def_validators:
-        return current_validators
+        return ref_validators
 
-    elif not current_validators:
+    elif not ref_validators:
         return def_validators
 
     else:
-        api_validators_mapping = _get_validators_mapping(def_validators)
-        test_validators_mapping = _get_validators_mapping(current_validators)
+        def_validators_mapping = _convert_validators_to_mapping(def_validators)
+        ref_validators_mapping = _convert_validators_to_mapping(ref_validators)
 
-        api_validators_mapping.update(test_validators_mapping)
-        return list(api_validators_mapping.values())
+        def_validators_mapping.update(ref_validators_mapping)
+        return list(def_validators_mapping.values())
 
 
-def _merge_extractor(def_extrators, current_extractors):
-    """ merge def_extrators with current_extractors
-    @params:
-        def_extrators: [{"var1": "val1"}, {"var2": "val2"}]
-        current_extractors: [{"var1": "val111"}, {"var3": "val3"}]
-    @return:
-        [
-            {"var1": "val111"},
-            {"var2": "val2"},
-            {"var3": "val3"}
-        ]
+def _merge_extractor(def_extrators, ref_extractors):
+    """ merge def_extrators with ref_extractors
+
+    Args:
+        def_extrators (list): [{"var1": "val1"}, {"var2": "val2"}]
+        ref_extractors (list): [{"var1": "val111"}, {"var3": "val3"}]
+
+    Returns:
+        list: merged extractors
+
+    Examples:
+        >>> def_extrators = [{"var1": "val1"}, {"var2": "val2"}]
+        >>> ref_extractors = [{"var1": "val111"}, {"var3": "val3"}]
+        >>> _merge_extractor(def_extrators, ref_extractors)
+            [
+                {"var1": "val111"},
+                {"var2": "val2"},
+                {"var3": "val3"}
+            ]
+
     """
     if not def_extrators:
-        return current_extractors
+        return ref_extractors
 
-    elif not current_extractors:
+    elif not ref_extractors:
         return def_extrators
 
     else:
@@ -687,7 +695,7 @@ def _merge_extractor(def_extrators, current_extractors):
             var_name = list(api_extrator.keys())[0]
             extractor_dict[var_name] = api_extrator[var_name]
 
-        for test_extrator in current_extractors:
+        for test_extrator in ref_extractors:
             if len(test_extrator) != 1:
                 logger.log_warning("incorrect extractor: {}".format(test_extrator))
                 continue
@@ -702,63 +710,6 @@ def _merge_extractor(def_extrators, current_extractors):
         return extractor_list
 
 
-def load_testcases(path):
-    """ load testcases from file path
-
-    Args:
-        path (str): testcase file/foler path.
-            path could be in several types:
-                - absolute/relative file path
-                - absolute/relative folder path
-                - list/set container with file(s) and/or folder(s)
-
-    Returns:
-        list: testcases list, each testcase is corresponding to a file
-        [
-            testcase_dict_1,
-            testcase_dict_2
-        ]
-    """
-    if isinstance(path, (list, set)):
-        testcases_list = []
-
-        for file_path in set(path):
-            testcases = load_testcases(file_path)
-            if not testcases:
-                continue
-            testcases_list.extend(testcases)
-
-        return testcases_list
-
-    if not os.path.isabs(path):
-        path = os.path.join(os.getcwd(), path)
-
-    if path in testcases_cache_mapping:
-        return testcases_cache_mapping[path]
-
-    if os.path.isdir(path):
-        files_list = load_folder_files(path)
-        testcases_list = load_testcases(files_list)
-
-    elif os.path.isfile(path):
-        try:
-            testcase = _load_test_file(path)
-            if testcase["testcases"]:
-                testcases_list = [testcase]
-            else:
-                testcases_list = []
-        except exceptions.FileFormatError:
-            testcases_list = []
-
-    else:
-        err_msg = "path not exist: {}".format(path)
-        logger.log_error(err_msg)
-        raise exceptions.FileNotFound(err_msg)
-
-    testcases_cache_mapping[path] = testcases_list
-    return testcases_list
-
-
 def load_folder_content(folder_path):
     """ load api/testcases/testsuites definitions from folder.
 
@@ -843,7 +794,7 @@ def load_api_folder(api_folder_path=None):
             api_dict["function_meta"] = function_meta
             api_definition_mapping[func_name] = api_dict
 
-    project_mapping["tests"]["api"] = api_definition_mapping
+    project_mapping["def-api"] = api_definition_mapping
     return api_definition_mapping
 
 
@@ -874,7 +825,7 @@ def load_test_folder(test_folder_path=None):
         dict: testcases definition mapping.
 
             {
-                "tests/testcases/setup.yml": [
+                "create_and_check": [
                     {"config": {}},
                     {"test": {}},
                     {"test": {}}
@@ -900,7 +851,7 @@ def load_test_folder(test_folder_path=None):
             "config": {
                 "path": test_file_path
             },
-            "tests": []
+            "teststeps": []
         }
         for item in items:
             key, block = item.popitem()
@@ -919,13 +870,13 @@ def load_test_folder(test_folder_path=None):
                 if func_name in test_definition_mapping:
                     logger.log_warning("API definition duplicated: {}".format(func_name))
 
-                block["function_meta"] = function_meta
+                testcase["function_meta"] = function_meta
                 test_definition_mapping[func_name] = testcase
             else:
                 # key == "test":
-                testcase["tests"].append(block)
+                testcase["teststeps"].append(block)
 
-    project_mapping["tests"]["testcases"] = test_definition_mapping
+    project_mapping["def-testcase"] = test_definition_mapping
     return test_definition_mapping
 
 
@@ -949,15 +900,59 @@ def load_project_tests(folder_path=None):
     return project_mapping
 
 
-def load(path):
-    """ main interface for loading testcases
+def load_testcases(path):
+    """ load testcases from file path
 
     Args:
-        path (str): testcase file/folder path
+        path (str): testcase file/foler path.
+            path could be in several types:
+                - absolute/relative file path
+                - absolute/relative folder path
+                - list/set container with file(s) and/or folder(s)
 
     Returns:
-        list: testcases list
+        list: testcases list, each testcase is corresponding to a file
+        [
+            testcase_dict_1,
+            testcase_dict_2
+        ]
 
     """
-    _load_test_dependencies()
-    return load_testcases(path)
+    if isinstance(path, (list, set)):
+        testcases_list = []
+
+        for file_path in set(path):
+            testcases = load_testcases(file_path)
+            if not testcases:
+                continue
+            testcases_list.extend(testcases)
+
+        return testcases_list
+
+    if not os.path.isabs(path):
+        path = os.path.join(os.getcwd(), path)
+
+    if path in testcases_cache_mapping:
+        return testcases_cache_mapping[path]
+
+    if os.path.isdir(path):
+        files_list = load_folder_files(path)
+        testcases_list = load_testcases(files_list)
+
+    elif os.path.isfile(path):
+        try:
+            testcase = _load_test_file(path)
+            if testcase["teststeps"]:
+                testcases_list = [testcase]
+            else:
+                testcases_list = []
+        except exceptions.FileFormatError:
+            testcases_list = []
+
+    else:
+        err_msg = "path not exist: {}".format(path)
+        logger.log_error(err_msg)
+        raise exceptions.FileNotFound(err_msg)
+
+    testcases_cache_mapping[path] = testcases_list
+    return testcases_list
diff --git a/httprunner/locusts.py b/httprunner/locusts.py
index 5b228a81..4f7d2adb 100644
--- a/httprunner/locusts.py
+++ b/httprunner/locusts.py
@@ -40,7 +40,7 @@ def gen_locustfile(testcase_file_path):
         "templates",
         "locustfile_template"
     )
-    testcases = loader.load(testcase_file_path)
+    testcases = loader.load_testcases(testcase_file_path)
     host = testcases[0].get("config", {}).get("request", {}).get("base_url", "")
 
     with io.open(template_path, encoding='utf-8') as template:
diff --git a/httprunner/task.py b/httprunner/task.py
index c5e57951..d9e7494b 100644
--- a/httprunner/task.py
+++ b/httprunner/task.py
@@ -34,39 +34,39 @@ class TestCase(unittest.TestCase):
 
 
 class TestSuite(unittest.TestSuite):
-    """ create test suite with a testset, it may include one or several testcases.
-        each suite should initialize a separate Runner() with testset config.
-    @param
-        (dict) testset
+    """ create test suite with a testcase, it may include one or several teststeps.
+        each suite should initialize a separate Runner() with testcase config.
+
+    Args:
+        testcase (dict): testcase dict
             {
-                "name": "testset description",
                 "config": {
-                    "name": "testset description",
+                    "name": "testcase description",
                     "parameters": {},
                     "variables": [],
                     "request": {},
                     "output": []
                 },
-                "testcases": [
+                "teststeps": [
                     {
-                        "name": "testcase description",
+                        "name": "teststep1 description",
                         "parameters": {},
                         "variables": [],    # optional, override
                         "request": {},
                         "extract": {},      # optional
                         "validate": {}      # optional
                     },
-                    testcase12
+                    teststep2
                 ]
             }
-        (dict) variables_mapping:
-            passed in variables mapping, it will override variables in config block
+        variables_mapping (dict): passed in variables mapping, it will override variables in config block.
+
     """
-    def __init__(self, testset, variables_mapping=None, http_client_session=None):
+    def __init__(self, testcase, variables_mapping=None, http_client_session=None):
         super(TestSuite, self).__init__()
         self.test_runner_list = []
 
-        self.config = testset.get("config", {})
+        self.config = testcase.get("config", {})
         self.output_variables_list = self.config.get("output", [])
         self.testset_file_path = self.config.get("path")
         config_dict_parameters = self.config.get("parameters", [])
@@ -80,22 +80,22 @@ class TestSuite(unittest.TestSuite):
             config_dict_parameters
         )
         self.testcase_parser = context.TestcaseParser()
-        testcases = testset.get("testcases", [])
+        teststeps = testcase.get("teststeps", [])
 
         for config_variables in config_parametered_variables_list:
             # config level
             self.config["variables"] = config_variables
             test_runner = runner.Runner(self.config, http_client_session)
 
-            for testcase_dict in testcases:
-                testcase_dict = copy.copy(testcase_dict)
+            for teststep_dict in teststeps:
+                teststep_dict = copy.copy(teststep_dict)
                 # testcase level
                 testcase_parametered_variables_list = self._get_parametered_variables(
-                    testcase_dict.get("variables", []),
-                    testcase_dict.get("parameters", [])
+                    teststep_dict.get("variables", []),
+                    teststep_dict.get("parameters", [])
                 )
                 for testcase_variables in testcase_parametered_variables_list:
-                    testcase_dict["variables"] = testcase_variables
+                    teststep_dict["variables"] = testcase_variables
 
                     # eval testcase name with bind variables
                     variables = utils.override_variables_binds(
@@ -104,13 +104,13 @@ class TestSuite(unittest.TestSuite):
                     )
                     self.testcase_parser.update_binded_variables(variables)
                     try:
-                        testcase_name = self.testcase_parser.eval_content_with_bindings(testcase_dict["name"])
+                        testcase_name = self.testcase_parser.eval_content_with_bindings(teststep_dict["name"])
                     except (AssertionError, exceptions.ParamsError):
-                        logger.log_warning("failed to eval testcase name: {}".format(testcase_dict["name"]))
-                        testcase_name = testcase_dict["name"]
+                        logger.log_warning("failed to eval testcase name: {}".format(teststep_dict["name"]))
+                        testcase_name = teststep_dict["name"]
                     self.test_runner_list.append((test_runner, variables))
 
-                    self._add_test_to_suite(testcase_name, test_runner, testcase_dict)
+                    self._add_test_to_suite(testcase_name, test_runner, teststep_dict)
 
     def _get_parametered_variables(self, variables, parameters):
         """ parameterize varaibles with parameters
@@ -166,15 +166,14 @@ def init_test_suites(path_or_testcases, mapping=None, http_client_session=None):
     Args:
         path_or_testcases (str/dict/list): testcase file path or testcase dict or testcases list
 
-            testset_dict
+            testcase_dict
             or
             [
-                testset_dict_1,
-                testset_dict_2,
+                testcase_dict_1,
+                testcase_dict_2,
                 {
                     "config": {},
-                    "api": {},
-                    "testcases": [testcase11, testcase12]
+                    "teststeps": [teststep11, teststep12]
                 }
             ]
 
@@ -188,7 +187,7 @@ def init_test_suites(path_or_testcases, mapping=None, http_client_session=None):
     if validator.is_testcases(path_or_testcases):
         testcases = path_or_testcases
     else:
-        testcases = loader.load(path_or_testcases)
+        testcases = loader.load_testcases(path_or_testcases)
 
     # TODO: move comparator uniform here
     mapping = mapping or {}
@@ -237,12 +236,12 @@ class HttpRunner(object):
         """ start to run test with varaibles mapping.
 
         Args:
-            path_or_testcases (str/list/dict): YAML/JSON testset file path or testset list
+            path_or_testcases (str/list/dict): YAML/JSON testcase file path or testcase list
                 path: path could be in several type
                     - absolute/relative file path
                     - absolute/relative folder path
                     - list/set container with file(s) and/or folder(s)
-                testsets: testset or list of testset
+                testcases: testcase dict or list of testcases
                     - (dict) testset_dict
                     - (list) list of testset_dict
                         [
diff --git a/httprunner/validator.py b/httprunner/validator.py
index e47553f1..bbd7fcf6 100644
--- a/httprunner/validator.py
+++ b/httprunner/validator.py
@@ -6,35 +6,48 @@ TODO: refactor with JSON schema validate
 """
 
 def is_testcase(data_structure):
-    """ check if data_structure is a testcase
-    testcase should always be in the following data structure:
-        {
-            "name": "desc1",
-            "config": {},
-            "api": {},
-            "testcases": [testcase11, testcase12]
-        }
+    """ check if data_structure is a testcase.
+
+    Args:
+        data_structure (dict): testcase should always be in the following data structure:
+
+            {
+                "name": "desc1",
+                "config": {},
+                "api": {},
+                "testcases": [testcase11, testcase12]
+            }
+
+    Returns:
+        bool: True if data_structure is valid testcase, otherwise False.
+
     """
     if not isinstance(data_structure, dict):
         return False
 
-    if "name" not in data_structure or "testcases" not in data_structure:
+    if "name" not in data_structure or "teststeps" not in data_structure:
         return False
 
-    if not isinstance(data_structure["testcases"], list):
+    if not isinstance(data_structure["teststeps"], list):
         return False
 
     return True
 
 def is_testcases(data_structure):
-    """ check if data_structure is testcase or testcases list
-    testsets should always be in the following data structure:
-        testset_dict
-        or
-        [
-            testset_dict_1,
-            testset_dict_2
-        ]
+    """ check if data_structure is testcase or testcases list.
+
+    Args:
+        data_structure (dict): testcase(s) should always be in the following data structure:
+
+            testcase_dict
+            or
+            [
+                testcase_dict_1,
+                testcase_dict_2
+            ]
+    Returns:
+        bool: True if data_structure is valid testcase(s), otherwise False.
+
     """
     if not isinstance(data_structure, list):
         return is_testcase(data_structure)
diff --git a/tests/test_httprunner.py b/tests/test_httprunner.py
index 9c5002e6..85c53033 100644
--- a/tests/test_httprunner.py
+++ b/tests/test_httprunner.py
@@ -23,7 +23,7 @@ class TestHttpRunner(ApiServerUnittest):
                 'output': ['token']
             },
             'api': {},
-            'testcases': [
+            'teststeps': [
                 {
                     'name': '/api/get-token',
                     'request': {
@@ -114,7 +114,7 @@ class TestHttpRunner(ApiServerUnittest):
         testsets = [
             {
                 "name": "post data",
-                "testcases": [
+                "teststeps": [
                     {
                         "name": "post data",
                         "request": {
diff --git a/tests/test_loader.py b/tests/test_loader.py
index fb491d5f..0cf7161a 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -219,60 +219,34 @@ class TestModuleLoader(unittest.TestCase):
 
 class TestSuiteLoader(unittest.TestCase):
 
-    def setUp(self):
-        loader.overall_def_dict = {
-            "api": {},
-            "suite": {}
-        }
-
-    def test_load_test_dependencies(self):
-        loader._load_test_dependencies()
-        overall_def_dict = loader.overall_def_dict
-        self.assertIn("get_token", overall_def_dict["api"])
-        self.assertIn("create_and_check", overall_def_dict["suite"])
-
-    def test_load_api_file(self):
-        loader._load_api_file("tests/api/basic.yml")
-        overall_api_def_dict = loader.overall_def_dict["api"]
-        self.assertIn("get_token",overall_api_def_dict)
-        self.assertEqual("/api/get-token", overall_api_def_dict["get_token"]["request"]["url"])
-        self.assertIn("$user_agent", overall_api_def_dict["get_token"]["function_meta"]["args"])
-        self.assertEqual(len(overall_api_def_dict["get_token"]["validate"]), 3)
-
-    def test_load_test_file_suite(self):
-        loader._load_api_file("tests/api/basic.yml")
-        testset = loader._load_test_file("tests/suite/create_and_get.yml")
-        self.assertEqual(testset["config"]["name"], "create user and check result.")
-        self.assertEqual(len(testset["testcases"]), 3)
-        self.assertEqual(testset["testcases"][0]["name"], "make sure user $uid does not exist")
-        self.assertEqual(testset["testcases"][0]["request"]["url"], "/api/users/$uid")
+    @classmethod
+    def setUpClass(cls):
+        project_dir = os.path.join(os.getcwd(), "tests")
+        loader.load_project_tests(project_dir)
 
     def test_load_test_file_testcase(self):
-        loader._load_test_dependencies()
-        testset = loader._load_test_file("tests/testcases/smoketest.yml")
-        self.assertEqual(testset["config"]["name"], "smoketest")
-        self.assertEqual(testset["config"]["path"], "tests/testcases/smoketest.yml")
-        self.assertIn("device_sn", testset["config"]["variables"][0])
-        self.assertEqual(len(testset["testcases"]), 8)
-        self.assertEqual(testset["testcases"][0]["name"], "get token")
+        testcase = loader._load_test_file("tests/testcases/smoketest.yml")
+        self.assertEqual(testcase["config"]["name"], "smoketest")
+        self.assertEqual(testcase["config"]["path"], "tests/testcases/smoketest.yml")
+        self.assertIn("device_sn", testcase["config"]["variables"][0])
+        self.assertEqual(len(testcase["teststeps"]), 8)
+        self.assertEqual(testcase["teststeps"][0]["name"], "get token")
 
     def test_get_block_by_name(self):
-        loader._load_test_dependencies()
         ref_call = "get_user($uid, $token)"
-        block = loader._get_block_by_name(ref_call, "api")
+        block = loader._get_block_by_name(ref_call, "def-api")
         self.assertEqual(block["request"]["url"], "/api/users/$uid")
         self.assertEqual(block["function_meta"]["func_name"], "get_user")
         self.assertEqual(block["function_meta"]["args"], ['$uid', '$token'])
 
     def test_get_block_by_name_args_mismatch(self):
-        loader._load_test_dependencies()
         ref_call = "get_user($uid, $token, $var)"
         with self.assertRaises(exceptions.ParamsError):
-            loader._get_block_by_name(ref_call, "api")
+            loader._get_block_by_name(ref_call, "def-api")
 
     def test_override_block(self):
-        loader._load_test_dependencies()
-        def_block = loader._get_block_by_name("get_token($user_agent, $device_sn, $os_platform, $app_version)", "api")
+        def_block = loader._get_block_by_name(
+            "get_token($user_agent, $device_sn, $os_platform, $app_version)", "def-api")
         test_block = {
             "name": "override block",
             "variables": [
@@ -286,28 +260,26 @@ class TestSuiteLoader(unittest.TestCase):
             ]
         }
 
-        loader._override_block(def_block, test_block)
+        loader._extend_block(test_block, def_block)
         self.assertEqual(test_block["name"], "override block")
         self.assertIn({'check': 'status_code', 'expect': 201, 'comparator': 'eq'}, test_block["validate"])
         self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, test_block["validate"])
 
     def test_get_test_definition_api(self):
-        loader._load_test_dependencies()
-        api_def = loader._get_test_definition("get_headers", "api")
+        api_def = loader._get_test_definition("get_headers", "def-api")
         self.assertEqual(api_def["request"]["url"], "/headers")
         self.assertEqual(len(api_def["setup_hooks"]), 2)
         self.assertEqual(len(api_def["teardown_hooks"]), 1)
 
         with self.assertRaises(exceptions.ApiNotFound):
-            loader._get_test_definition("get_token_XXX", "api")
+            loader._get_test_definition("get_token_XXX", "def-api")
 
     def test_get_test_definition_suite(self):
-        loader._load_test_dependencies()
-        api_def = loader._get_test_definition("create_and_check", "suite")
+        api_def = loader._get_test_definition("create_and_check", "def-testcase")
         self.assertEqual(api_def["config"]["name"], "create user and check result.")
 
         with self.assertRaises(exceptions.TestcaseNotFound):
-            loader._get_test_definition("create_and_check_XXX", "suite")
+            loader._get_test_definition("create_and_check_XXX", "def-testcase")
 
     def test_merge_validator(self):
         def_validators = [
@@ -376,7 +348,7 @@ class TestSuiteLoader(unittest.TestCase):
         self.assertEqual(len(testset_list), 1)
         self.assertIn("path", testset_list[0]["config"])
         self.assertEqual(testset_list[0]["config"]["path"], path)
-        self.assertEqual(len(testset_list[0]["testcases"]), 3)
+        self.assertEqual(len(testset_list[0]["teststeps"]), 3)
         testsets_list.extend(testset_list)
 
         # relative file path
@@ -385,7 +357,7 @@ class TestSuiteLoader(unittest.TestCase):
         self.assertEqual(len(testset_list), 1)
         self.assertIn("path", testset_list[0]["config"])
         self.assertIn(path, testset_list[0]["config"]["path"])
-        self.assertEqual(len(testset_list[0]["testcases"]), 3)
+        self.assertEqual(len(testset_list[0]["teststeps"]), 3)
         testsets_list.extend(testset_list)
 
         # list/set container with file(s)
@@ -395,20 +367,19 @@ class TestSuiteLoader(unittest.TestCase):
         ]
         testset_list = loader.load_testcases(path)
         self.assertEqual(len(testset_list), 2)
-        self.assertEqual(len(testset_list[0]["testcases"]), 3)
-        self.assertEqual(len(testset_list[1]["testcases"]), 3)
+        self.assertEqual(len(testset_list[0]["teststeps"]), 3)
+        self.assertEqual(len(testset_list[1]["teststeps"]), 3)
         testsets_list.extend(testset_list)
         self.assertEqual(len(testsets_list), 4)
 
         for testset in testsets_list:
-            for test in testset["testcases"]:
+            for test in testset["teststeps"]:
                 self.assertIn('name', test)
                 self.assertIn('request', test)
                 self.assertIn('url', test['request'])
                 self.assertIn('method', test['request'])
 
     def test_load_testcases_by_path_folder(self):
-        loader._load_test_dependencies()
         # absolute folder path
         path = os.path.join(os.getcwd(), 'tests/data')
         testset_list_1 = loader.load_testcases(path)
@@ -447,15 +418,14 @@ class TestSuiteLoader(unittest.TestCase):
             loader.load_testcases(path)
 
     def test_load_testcases_by_path_layered(self):
-        loader._load_test_dependencies()
         path = os.path.join(
             os.getcwd(), 'tests/data/demo_testset_layer.yml')
         testsets_list = loader.load_testcases(path)
         self.assertIn("variables", testsets_list[0]["config"])
         self.assertIn("request", testsets_list[0]["config"])
-        self.assertIn("request", testsets_list[0]["testcases"][0])
-        self.assertIn("url", testsets_list[0]["testcases"][0]["request"])
-        self.assertIn("validate", testsets_list[0]["testcases"][0])
+        self.assertIn("request", testsets_list[0]["teststeps"][0])
+        self.assertIn("url", testsets_list[0]["teststeps"][0]["request"])
+        self.assertIn("validate", testsets_list[0]["teststeps"][0])
 
     def test_load_folder_content(self):
         path = os.path.join(os.getcwd(), "tests", "api")
@@ -481,6 +451,10 @@ class TestSuiteLoader(unittest.TestCase):
             testcases_definition_mapping["setup_and_reset"]["config"]["name"],
             "setup and reset all."
         )
+        self.assertEqual(
+            testcases_definition_mapping["setup_and_reset"]["function_meta"]["func_name"],
+            "setup_and_reset"
+        )
 
     def test_load_testsuites_folder(self):
         path = os.path.join(os.getcwd(), "tests", "testcases")
@@ -500,18 +474,18 @@ class TestSuiteLoader(unittest.TestCase):
         project_dir = os.path.join(os.getcwd(), "tests")
         project_tests = loader.load_project_tests(project_dir)
         self.assertEqual(project_tests["debugtalk"]["variables"]["SECRET_KEY"], "DebugTalk")
-        self.assertIn("get_token", project_tests["tests"]["api"])
-        self.assertIn("setup_and_reset", project_tests["tests"]["testcases"])
+        self.assertIn("get_token", project_tests["def-api"])
+        self.assertIn("setup_and_reset", project_tests["def-testcase"])
 
     def test_loader(self):
         hrunner = task.HttpRunner(dot_env_path="tests/data/test.env")
         self.assertEqual(hrunner.project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH")
         self.assertIn("debugtalk", hrunner.project_mapping)
-        self.assertIn("setup_and_reset", hrunner.project_mapping["tests"]["testcases"])
+        self.assertIn("setup_and_reset", hrunner.project_mapping["def-testcase"])
         self.assertEqual(
             hrunner.project_mapping["debugtalk"]["variables"]["SECRET_KEY"],
             "DebugTalk"
         )
         self.assertIn("get_sign", hrunner.project_mapping["debugtalk"]["functions"])
-        self.assertIn("get_token", hrunner.project_mapping["tests"]["api"])
-        self.assertIn("setup_and_reset", hrunner.project_mapping["tests"]["testcases"])
+        self.assertIn("get_token", hrunner.project_mapping["def-api"])
+        self.assertIn("setup_and_reset", hrunner.project_mapping["def-testcase"])
diff --git a/tests/test_runner.py b/tests/test_runner.py
index 82df9a48..1e44b968 100644
--- a/tests/test_runner.py
+++ b/tests/test_runner.py
@@ -170,7 +170,7 @@ class TestRunner(ApiServerUnittest):
                 "config": {
                     'path': 'tests/httpbin/hooks.yml',
                 },
-                "testcases": [
+                "teststeps": [
                     {
                         "name": "test teardown hooks",
                         "request": {
@@ -206,7 +206,7 @@ class TestRunner(ApiServerUnittest):
                 "config": {
                     'path': 'tests/httpbin/hooks.yml',
                 },
-                "testcases": [
+                "teststeps": [
                     {
                         "name": "test teardown hooks",
                         "request": {
@@ -236,7 +236,7 @@ class TestRunner(ApiServerUnittest):
                 "config": {
                     'path': 'tests/httpbin/hooks.yml',
                 },
-                "testcases": [
+                "teststeps": [
                     {
                         "name": "test teardown hooks",
                         "request": {
@@ -384,7 +384,7 @@ class TestRunner(ApiServerUnittest):
         testsets = loader.load_testcases(testcase_file_path)
         testset = testsets[0]
         config_dict_headers = testset["config"]["request"]["headers"]
-        test_dict_headers = testset["testcases"][0]["request"]["headers"]
+        test_dict_headers = testset["teststeps"][0]["request"]["headers"]
         headers = deep_update_dict(
             config_dict_headers,
             test_dict_headers
diff --git a/tests/test_task.py b/tests/test_task.py
index db2112c4..59b34494 100644
--- a/tests/test_task.py
+++ b/tests/test_task.py
@@ -37,7 +37,7 @@ class TestTask(ApiServerUnittest):
                     'output': ['token']
                 },
                 'api': {},
-                'testcases': [
+                'teststeps': [
                     {
                         'name': '/api/get-token',
                         'request': {