Merge pull request #398 from HttpRunner/pipline

refactor pipeline

.travis.yml

@@ -5,9 +5,10 @@ python:
   - 3.4
   - 3.5
   - 3.6
+  - 3.7-dev
 install:
   - pip install pipenv --upgrade-strategy=only-if-needed
-  - pipenv install --dev
+  - pipenv install --dev --skip-lock
 script:
   - pipenv run python setup.py install
   - pipenv run coverage run --source=httprunner -m unittest discover

httprunner/api.py

@@ -1,6 +1,5 @@
 # encoding: utf-8
 
-import copy
 import os
 import unittest
 
@@ -20,7 +19,6 @@ class HttpRunner(object):
             resultclass (class): HtmlTestResult or TextTestResult
             failfast (bool): False/True, stop the test run on the first error or failure.
             http_client_session (instance): requests.Session(), or locust.client.Session() instance.
-            dot_env_path (str): .env file path.
 
         Attributes:
             project_mapping (dict): save project loaded api/testcases, environments and debugtalk.py module.
@@ -36,158 +34,23 @@ class HttpRunner(object):
 
         """
         self.exception_stage = "initialize HttpRunner()"
-        loader.reset_loader()
-        loader.dot_env_path = kwargs.pop("dot_env_path", None)
         self.http_client_session = kwargs.pop("http_client_session", None)
-        self.kwargs = kwargs
+        kwargs.setdefault("resultclass", report.HtmlTestResult)
+        self.unittest_runner = unittest.TextTestRunner(**kwargs)
+        self.test_loader = unittest.TestLoader()
+        self.summary = None
 
-    def load_tests(self, path_or_testcases):
+    def _add_tests(self, testcases):
-        """ load testcases, extend and merge with api/testcase definitions.
+        """ initialize testcase with Runner() and add to test suite.
 
         Args:
-            path_or_testcases (str/dict/list): YAML/JSON testcase file path or testcase list
+            testcases (list): parsed testcases list
-                path (str): testcase file/folder path
-                testcases (dict/list): testcase dict or list of testcases
 
         Returns:
-            list: valid testcases list.
+            tuple: unittest.TestSuite()
-
-                [
-                    # testcase data structure
-                    {
-                        "config": {
-                            "name": "desc1",
-                            "path": "",         # optional
-                            "variables": [],    # optional
-                            "request": {}       # optional
-                        },
-                        "teststeps": [
-                            # teststep data structure
-                            {
-                                'name': 'test step desc2',
-                                'variables': [],    # optional
-                                'extract': [],      # optional
-                                'validate': [],
-                                'request': {},
-                                'function_meta': {}
-                            },
-                            teststep2   # another teststep dict
-                        ]
-                    },
-                    {}  # another testcase dict
-                ]
 
         """
-        self.exception_stage = "load tests"
+        def _add_teststep(test_runner, config, teststep_dict):
-        if validator.is_testcases(path_or_testcases):
-            # TODO: refactor
-            if isinstance(path_or_testcases, list):
-                for testcase in path_or_testcases:
-                    try:
-                        test_path = os.path.dirname(testcase["config"]["path"])
-                    except KeyError:
-                        test_path = os.getcwd()
-                    loader.load_project_tests(test_path)
-            else:
-                try:
-                    test_path = os.path.dirname(path_or_testcases["config"]["path"])
-                except KeyError:
-                    test_path = os.getcwd()
-                loader.load_project_tests(test_path)
-
-            testcases = path_or_testcases
-        else:
-            testcases = loader.load_testcases(path_or_testcases)
-
-        self.project_mapping = loader.project_mapping
-
-        if not testcases:
-            raise exceptions.TestcaseNotFound
-
-        if isinstance(testcases, dict):
-            testcases = [testcases]
-
-        return testcases
-
-    def parse_tests(self, testcases, variables_mapping=None):
-        """ parse testcases configs, including variables/parameters/name/request.
-
-        Args:
-            testcases (list): testcase list, with config unparsed.
-            variables_mapping (dict): if variables_mapping is specified, it will override variables in config block.
-
-        Returns:
-            list: parsed testcases list, with config variables/parameters/name/request parsed.
-
-        """
-        self.exception_stage = "parse tests"
-        variables_mapping = variables_mapping or {}
-
-        parsed_testcases_list = []
-        for testcase in testcases:
-            # parse config parameters
-            config_parameters = testcase.setdefault("config", {}).pop("parameters", [])
-            cartesian_product_parameters_list = parser.parse_parameters(
-                config_parameters,
-                self.project_mapping["debugtalk"]["variables"],
-                self.project_mapping["debugtalk"]["functions"]
-            ) or [{}]
-
-            for parameter_mapping in cartesian_product_parameters_list:
-                testcase_dict = copy.deepcopy(testcase)
-                config = testcase_dict.setdefault("config", {})
-
-                # parse config variables
-                raw_config_variables = config.get("variables", [])
-                parsed_config_variables = parser.parse_data(
-                    raw_config_variables,
-                    self.project_mapping["debugtalk"]["variables"],
-                    self.project_mapping["debugtalk"]["functions"]
-                )
-
-                # priority: passed in > debugtalk.py > parameters > variables
-                # override variables mapping with parameters mapping
-                config_variables = utils.override_mapping_list(
-                    parsed_config_variables, parameter_mapping)
-                # merge debugtalk.py module variables
-                config_variables.update(self.project_mapping["debugtalk"]["variables"])
-                # override variables mapping with passed in variables_mapping
-                config_variables = utils.override_mapping_list(
-                    config_variables, variables_mapping)
-
-                testcase_dict["config"]["variables"] = config_variables
-
-                # parse config name
-                testcase_dict["config"]["name"] = parser.parse_data(
-                    testcase_dict["config"].get("name", ""),
-                    config_variables,
-                    self.project_mapping["debugtalk"]["functions"]
-                )
-
-                # parse config request
-                testcase_dict["config"]["request"] = parser.parse_data(
-                    testcase_dict["config"].get("request", {}),
-                    config_variables,
-                    self.project_mapping["debugtalk"]["functions"]
-                )
-
-                # put loaded project functions to config
-                testcase_dict["config"]["functions"] = self.project_mapping["debugtalk"]["functions"]
-                parsed_testcases_list.append(testcase_dict)
-
-        return parsed_testcases_list
-
-    def initialize(self, testcases):
-        """ initialize test runner with parsed testcases.
-
-        Args:
-            testcases (list): testcases list
-
-        Returns:
-            tuple: (unittest.TextTestRunner(), unittest.TestSuite())
-
-        """
-        def __add_teststep(test_runner, config, teststep_dict):
             """ add teststep to testcase.
             """
             def test(self):
@@ -213,13 +76,7 @@ class HttpRunner(object):
             test.__doc__ = teststep_dict["name"]
             return test
 
-        self.exception_stage = "initialize unittest Runner() and TestSuite()"
+        test_suite = unittest.TestSuite()
-        self.kwargs.setdefault("resultclass", report.HtmlTestResult)
-        unittest_runner = unittest.TextTestRunner(**self.kwargs)
-
-        testcases_list = []
-        loader = unittest.TestLoader()
-        loaded_testcases = []
         for testcase in testcases:
             config = testcase.get("config", {})
             test_runner = runner.Runner(config, self.http_client_session)
@@ -231,49 +88,45 @@ class HttpRunner(object):
                     # suppose one testcase should not have more than 9999 steps,
                     # and one step should not run more than 999 times.
                     test_method_name = 'test_{:04}_{:03}'.format(index, times_index)
-                    test_method = __add_teststep(test_runner, config, teststep_dict)
+                    test_method = _add_teststep(test_runner, config, teststep_dict)
                     setattr(TestSequense, test_method_name, test_method)
 
-            loaded_testcase = loader.loadTestsFromTestCase(TestSequense)
+            loaded_testcase = self.test_loader.loadTestsFromTestCase(TestSequense)
             setattr(loaded_testcase, "config", config)
             setattr(loaded_testcase, "teststeps", testcase.get("teststeps", []))
             setattr(loaded_testcase, "runner", test_runner)
-            loaded_testcases.append(loaded_testcase)
+            test_suite.addTest(loaded_testcase)
 
-        test_suite = unittest.TestSuite(loaded_testcases)
+        return test_suite
-        return (unittest_runner, test_suite)
 
-    def run_tests(self, unittest_runner, test_suite):
+    def _run_suite(self, test_suite):
-        """ run tests with unittest_runner and test_suite
+        """ run tests in test_suite
 
         Args:
-            unittest_runner: unittest.TextTestRunner()
             test_suite: unittest.TestSuite()
 
         Returns:
             list: tests_results
 
         """
-        self.exception_stage = "running tests"
         tests_results = []
 
         for testcase in test_suite:
             testcase_name = testcase.config.get("name")
             logger.log_info("Start to run testcase: {}".format(testcase_name))
 
-            result = unittest_runner.run(testcase)
+            result = self.unittest_runner.run(testcase)
             tests_results.append((testcase, result))
 
         return tests_results
 
-    def aggregate(self, tests_results):
+    def _aggregate(self, tests_results):
         """ aggregate results
 
         Args:
             tests_results (list): list of (testcase, result)
 
         """
-        self.exception_stage = "aggregate results"
         self.summary = {
             "success": True,
             "stat": {},
@@ -299,45 +152,89 @@ class HttpRunner(object):
 
             self.summary["details"].append(testcase_summary)
 
-    def run(self, path_or_testcases, mapping=None):
+    def _run_tests(self, testcases, mapping=None):
         """ start to run test with variables mapping.
 
         Args:
-            path_or_testcases (str/list/dict): YAML/JSON testcase file path or testcase list
+            testcases (list): list of testcase_dict, each testcase is corresponding to a YAML/JSON file
-                path: path could be in several type
+                [
-                    - absolute/relative file path
+                    {   # testcase data structure
-                    - absolute/relative folder path
+                        "config": {
-                    - list/set container with file(s) and/or folder(s)
+                            "name": "desc1",
-                testcases: testcase dict or list of testcases
+                            "path": "testcase1_path",
-                    - (dict) testset_dict
+                            "variables": [],        # optional
-                    - (list) list of testset_dict
+                            "request": {}           # optional
-                        [
+                            "refs": {
-                            testset_dict_1,
+                                "debugtalk": {
-                            testset_dict_2
+                                    "variables": {},
+                                    "functions": {}
+                                },
+                                "env": {},
+                                "def-api": {},
+                                "def-testcase": {}
+                            }
+                        },
+                        "teststeps": [
+                            # teststep data structure
+                            {
+                                'name': 'test step desc2',
+                                'variables': [],    # optional
+                                'extract': [],      # optional
+                                'validate': [],
+                                'request': {},
+                                'function_meta': {}
+                            },
+                            teststep2   # another teststep dict
                         ]
-            mapping (dict): if mapping specified, it will override variables in config block.
+                    },
+                    testcase_dict_2     # another testcase dict
+                ]
+            mapping (dict): if mapping is specified, it will override variables in config block.
 
         Returns:
             instance: HttpRunner() instance
 
         """
-        # loader
+        self.exception_stage = "parse tests"
-        testcases_list = self.load_tests(path_or_testcases)
+        parsed_testcases_list = parser.parse_tests(testcases, mapping)
 
-        # parser
+        self.exception_stage = "add tests to test suite"
-        parsed_testcases_list = self.parse_tests(testcases_list)
+        test_suite = self._add_tests(parsed_testcases_list)
 
-        # initialize
+        self.exception_stage = "run test suite"
-        unittest_runner, test_suite = self.initialize(parsed_testcases_list)
+        results = self._run_suite(test_suite)
 
-        # running tests
+        self.exception_stage = "aggregate results"
-        results = self.run_tests(unittest_runner, test_suite)
+        self._aggregate(results)
-
-        # aggregate
-        self.aggregate(results)
 
         return self
 
+    def run(self, path_or_testcases, dot_env_path=None, mapping=None):
+        """ main interface, run testcases with variables mapping.
+
+        Args:
+            path_or_testcases (str/list/dict): testcase file/foler path, or valid testcases.
+            dot_env_path (str): specified .env file path.
+            mapping (dict): if mapping is specified, it will override variables in config block.
+
+        Returns:
+            instance: HttpRunner() instance
+
+        """
+        self.exception_stage = "load tests"
+
+        if validator.is_testcases(path_or_testcases):
+            if isinstance(path_or_testcases, dict):
+                testcases = [path_or_testcases]
+            else:
+                testcases = path_or_testcases
+        elif validator.is_testcase_path(path_or_testcases):
+            testcases = loader.load_tests(path_or_testcases, dot_env_path)
+        else:
+            raise exceptions.ParamsError("invalid testcase path or testcases.")
+
+        return self._run_tests(testcases, mapping)
+
     def gen_html_report(self, html_report_name=None, html_report_template=None):
         """ generate html report and return report path.
 
@@ -349,6 +246,9 @@ class HttpRunner(object):
             str: generated html report path
 
         """
+        if not self.summary:
+            raise exceptions.MyBaseError("run method should be called before gen_html_report.")
+
         self.exception_stage = "generate report"
         return report.render_html_report(
             self.summary,

httprunner/cli.py

@@ -79,10 +79,12 @@ def main_hrun():
 
     try:
         runner = HttpRunner(
-            failfast=args.failfast,
+            failfast=args.failfast
+        )
+        runner.run(
+            args.testset_paths,
             dot_env_path=args.dot_env_path
         )
-        runner.run(args.testset_paths)
     except Exception:
         logger.log_error("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage))
         raise

httprunner/context.py

@@ -183,10 +183,7 @@ class Context(object):
         """
         # TODO: move comparator uniform to init_test_suites
         comparator = utils.get_uniform_comparator(validator_dict["comparator"])
-        validate_func = self.TESTCASE_SHARED_FUNCTIONS_MAPPING.get(comparator)
+        validate_func = parser.get_mapping_function(comparator, self.TESTCASE_SHARED_FUNCTIONS_MAPPING)
-
-        if not validate_func:
-            raise exceptions.FunctionNotFound("comparator not found: {}".format(comparator))
 
         check_item = validator_dict["check"]
         check_value = validator_dict["check_value"]

httprunner/loader.py

@@ -10,24 +10,6 @@ import yaml
 from httprunner import built_in, exceptions, logger, parser, utils, validator
 from httprunner.compat import OrderedDict
 
-sys.path.insert(0, os.getcwd())
-
-project_mapping = {
-    "debugtalk": {
-        "variables": {},
-        "functions": {}
-    },
-    "env": {},
-    "def-api": {},
-    "def-testcase": {}
-}
-""" dict: save project loaded api/testcases definitions, environments and debugtalk.py module.
-"""
-
-dot_env_path = None
-testcases_cache_mapping = {}
-project_working_directory = os.getcwd()
-
 
 ###############################################################################
 ##   file loader
@@ -161,9 +143,11 @@ def load_folder_files(folder_path, recursive=True):
     return file_list
 
 
-def load_dot_env_file():
+def load_dot_env_file(dot_env_path):
-    """ load .env file, .env file should be located in project working directory by default.
+    """ load .env file.
-        If dot_env_path is specified, it will be loaded instead.
+
+    Args:
+        dot_env_path (str): .env file path
 
     Returns:
         dict: environment variables mapping
@@ -175,21 +159,15 @@ def load_dot_env_file():
             }
 
     Raises:
-        exceptions.FileFormatError: If env file format is invalid.
+        exceptions.FileFormatError: If .env file format is invalid.
 
     """
-    path = dot_env_path or os.path.join(project_working_directory, ".env")
+    if not os.path.isfile(dot_env_path):
-    if not os.path.isfile(path):
+        raise exceptions.FileNotFound(".env file path is not exist.")
-        if dot_env_path:
-            logger.log_error(".env file not exist: {}".format(dot_env_path))
-            sys.exit(1)
-        else:
-            logger.log_debug(".env file not exist in: {}".format(project_working_directory))
-            return {}
 
-    logger.log_info("Loading environment variables from {}".format(path))
+    logger.log_info("Loading environment variables from {}".format(dot_env_path))
     env_variables_mapping = {}
-    with io.open(path, 'r', encoding='utf-8') as fp:
+    with io.open(dot_env_path, 'r', encoding='utf-8') as fp:
         for line in fp:
             # maxsplit=1
             if "=" in line:
@@ -201,9 +179,7 @@ def load_dot_env_file():
 
             env_variables_mapping[variable.strip()] = value.strip()
 
-    project_mapping["env"] = env_variables_mapping
     utils.set_os_environ(env_variables_mapping)
-
     return env_variables_mapping
 
 
@@ -281,13 +257,15 @@ def load_builtin_module():
     """ load built_in module
     """
     built_in_module = load_python_module(built_in)
-    project_mapping["debugtalk"] = built_in_module
+    return built_in_module
 
 
 def load_debugtalk_module():
-    """ load project debugtalk.py module and merge with builtin module.
+    """ load project debugtalk.py module
         debugtalk.py should be located in project working directory.
-        variables and functions mapping for debugtalk.py
+
+    Returns:
+        dict: debugtalk module mapping
             {
                 "variables": {},
                 "functions": {}
@@ -297,10 +275,7 @@ def load_debugtalk_module():
     # load debugtalk.py module
     imported_module = importlib.import_module("debugtalk")
     debugtalk_module = load_python_module(imported_module)
-
+    return debugtalk_module
-    # override built_in module with debugtalk.py module
-    project_mapping["debugtalk"]["variables"].update(debugtalk_module["variables"])
-    project_mapping["debugtalk"]["functions"].update(debugtalk_module["functions"])
 
 
 def get_module_item(module_mapping, item_type, item_name):
@@ -340,12 +315,11 @@ def get_module_item(module_mapping, item_type, item_name):
 ##   testcase loader
 ###############################################################################
 
-def _load_test_file(file_path):
+def _load_test_file(file_path, project_mapping):
     """ load testcase file or testsuite file
 
     Args:
         file_path (str): absolute valid file path. file_path should be in the following format:
-
             [
                 {
                     "config": {
@@ -376,6 +350,7 @@ def _load_test_file(file_path):
                     }
                 }
             ]
+        project_mapping (dict): project_mapping
 
     Returns:
         dict: testcase dict
@@ -406,7 +381,7 @@ def _load_test_file(file_path):
 
             def extend_api_definition(block):
                 ref_call = block["api"]
-                def_block = _get_block_by_name(ref_call, "def-api")
+                def_block = _get_block_by_name(ref_call, "def-api", project_mapping)
                 _extend_block(block, def_block)
 
             # reference api
@@ -417,7 +392,7 @@ def _load_test_file(file_path):
             # reference testcase
             elif "suite" in test_block: # TODO: replace suite with testcase
                 ref_call = test_block["suite"]
-                block = _get_block_by_name(ref_call, "def-testcase")
+                block = _get_block_by_name(ref_call, "def-testcase", project_mapping)
                 # TODO: bugfix lost block config variables
                 for teststep in block["teststeps"]:
                     if "api" in teststep:
@@ -436,13 +411,14 @@ def _load_test_file(file_path):
     return testcase
 
 
-def _get_block_by_name(ref_call, ref_type):
+def _get_block_by_name(ref_call, ref_type, project_mapping):
     """ get test content by reference name.
 
     Args:
         ref_call (str): call function.
             e.g. api_v1_Account_Login_POST($UserName, $Password)
         ref_type (enum): "def-api" or "def-testcase"
+        project_mapping (dict): project_mapping
 
     Returns:
         dict: api/testcase definition.
@@ -454,7 +430,7 @@ def _get_block_by_name(ref_call, ref_type):
     function_meta = parser.parse_function(ref_call)
     func_name = function_meta["func_name"]
     call_args = function_meta["args"]
-    block = _get_test_definition(func_name, ref_type)
+    block = _get_test_definition(func_name, ref_type, project_mapping)
     def_args = block.get("function_meta", {}).get("args", [])
 
     if len(call_args) != len(def_args):
@@ -477,12 +453,13 @@ def _get_block_by_name(ref_call, ref_type):
     return block
 
 
-def _get_test_definition(name, ref_type):
+def _get_test_definition(name, ref_type, project_mapping):
     """ get expected api or testcase.
 
     Args:
         name (str): api or testcase name
         ref_type (enum): "def-api" or "def-testcase"
+        project_mapping (dict): project_mapping
 
     Returns:
         dict: expected api/testcase info if found.
@@ -764,7 +741,6 @@ def load_api_folder(api_folder_path):
             api_dict["function_meta"] = function_meta
             api_definition_mapping[func_name] = api_dict
 
-    project_mapping["def-api"] = api_definition_mapping
     return api_definition_mapping
 
 
@@ -842,29 +818,9 @@ def load_test_folder(test_folder_path):
                 # key == "test":
                 testcase["teststeps"].append(block)
 
-    project_mapping["def-testcase"] = test_definition_mapping
     return test_definition_mapping
 
 
-def reset_loader():
-    """ reset project mapping.
-    """
-    global project_working_directory
-    project_working_directory = os.getcwd()
-
-    global dot_env_path
-    dot_env_path = None
-
-    project_mapping["debugtalk"] = {
-        "variables": {},
-        "functions": {}
-    }
-    project_mapping["env"] = {}
-    project_mapping["def-api"] = {}
-    project_mapping["def-testcase"] = {}
-    testcases_cache_mapping.clear()
-
-
 def locate_debugtalk_py(start_path):
     """ locate debugtalk.py file.
 
@@ -879,57 +835,99 @@ def locate_debugtalk_py(start_path):
         return None
 
 
-def load_project_tests(test_path):
+def load_project_tests(test_path, dot_env_path=None):
     """ load api, testcases, .env, builtin module and debugtalk.py.
         api/testcases folder is relative to project_working_directory
 
     Args:
         test_path (str): test file/folder path, locate pwd from this path.
+        dot_env_path (str): specified .env file path
+
+    Returns:
+        dict: project loaded api/testcases definitions, environments and debugtalk.py module.
 
     """
-    global project_working_directory
+    project_mapping = {}
-
-    reset_loader()
-    load_builtin_module()
 
     debugtalk_path = locate_debugtalk_py(test_path)
     # locate PWD with debugtalk.py path
     if debugtalk_path:
         # The folder contains debugtalk.py will be treated as PWD.
-        # add PWD to sys.path
         project_working_directory = os.path.dirname(debugtalk_path)
     else:
-        # debugtalk.py not found, use os.getcwd() as PWD.
+        # debugtalk.py is not found, use os.getcwd() as PWD.
         project_working_directory = os.getcwd()
 
+    # add PWD to sys.path
+    sys.path.insert(0, project_working_directory)
+
     # load .env
-    load_dot_env_file()
+    dot_env_path = dot_env_path or os.path.join(project_working_directory, ".env")
+    if os.path.isfile(dot_env_path):
+        project_mapping["env"] = load_dot_env_file(dot_env_path)
+    else:
+        project_mapping["env"] = {}
 
     # load debugtalk.py
     if debugtalk_path:
-        sys.path.insert(0, project_working_directory)
+        project_mapping["debugtalk"] = load_debugtalk_module()
-        load_debugtalk_module()
+    else:
+        project_mapping["debugtalk"] = {
+            "variables": {},
+            "functions": {}
+        }
 
-    load_api_folder(os.path.join(project_working_directory, "api"))
+    project_mapping["def-api"] = load_api_folder(os.path.join(project_working_directory, "api"))
     # TODO: replace suite with testcases
-    load_test_folder(os.path.join(project_working_directory, "suite"))
+    project_mapping["def-testcase"] = load_test_folder(os.path.join(project_working_directory, "suite"))
+
+    return project_mapping
 
 
-def load_testcases(path):
+def load_tests(path, dot_env_path=None):
     """ load testcases from file path, extend and merge with api/testcase definitions.
 
     Args:
-        path (str): testcase file/foler path.
+        path (str/list): testcase file/foler path.
             path could be in several types:
                 - absolute/relative file path
                 - absolute/relative folder path
                 - list/set container with file(s) and/or folder(s)
+        dot_env_path (str): specified .env file path
 
     Returns:
         list: testcases list, each testcase is corresponding to a file
         [
-            testcase_dict_1,
+            {   # testcase data structure
-            testcase_dict_2
+                "config": {
+                    "name": "desc1",
+                    "path": "testcase1_path",
+                    "variables": [],                    # optional
+                    "request": {}                       # optional
+                    "refs": {
+                        "debugtalk": {
+                            "variables": {},
+                            "functions": {}
+                        },
+                        "env": {},
+                        "def-api": {},
+                        "def-testcase": {}
+                    }
+                },
+                "teststeps": [
+                    # teststep data structure
+                    {
+                        'name': 'test step desc2',
+                        'variables': [],    # optional
+                        'extract': [],      # optional
+                        'validate': [],
+                        'request': {},
+                        'function_meta': {}
+                    },
+                    teststep2   # another teststep dict
+                ]
+            },
+            testcase_dict_2     # another testcase dict
         ]
 
     """
@@ -937,7 +935,7 @@ def load_testcases(path):
         testcases_list = []
 
         for file_path in set(path):
-            testcases = load_testcases(file_path)
+            testcases = load_tests(file_path, dot_env_path)
             if not testcases:
                 continue
             testcases_list.extend(testcases)
@@ -952,24 +950,18 @@ def load_testcases(path):
     if not os.path.isabs(path):
         path = os.path.join(os.getcwd(), path)
 
-    if path not in testcases_cache_mapping:
-        load_project_tests(path)
-    else:
-        return testcases_cache_mapping[path]
-
     if os.path.isdir(path):
         files_list = load_folder_files(path)
-        testcases_list = load_testcases(files_list)
+        testcases_list = load_tests(files_list, dot_env_path)
 
     elif os.path.isfile(path):
         try:
-            testcase = _load_test_file(path)
+            project_mapping = load_project_tests(path, dot_env_path)
-            if testcase["teststeps"]:
+            testcase = _load_test_file(path, project_mapping)
-                testcases_list = [testcase]
+            testcase["config"]["path"] = path
-            else:
+            testcase["config"]["refs"] = project_mapping
-                testcases_list = []
+            testcases_list = [testcase]
         except exceptions.FileFormatError:
             testcases_list = []
 
-    testcases_cache_mapping[path] = testcases_list
     return testcases_list

httprunner/locusts.py

@@ -40,7 +40,7 @@ def gen_locustfile(testcase_file_path):
         "templates",
         "locustfile_template"
     )
-    testcases = loader.load_testcases(testcase_file_path)
+    testcases = loader.load_tests(testcase_file_path)
     host = testcases[0].get("config", {}).get("request", {}).get("base_url", "")
 
     with io.open(template_path, encoding='utf-8') as template:

httprunner/logger.py

@@ -3,7 +3,7 @@
 import logging
 import sys
 
-from colorama import Back, Fore, Style, init
+from colorama import Fore, init
 from colorlog import ColoredFormatter
 
 init(autoreset=True)

httprunner/parser.py

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 import ast
+import copy
 import os
 import re
 
@@ -325,6 +326,34 @@ def parse_parameters(parameters, variables_mapping, functions_mapping):
 ##  parse content with variables and functions mapping
 ###############################################################################
 
+def get_builtin_item(item_type, item_name):
+    """
+
+    Args:
+        item_type (enum): "variables" or "functions"
+        item_name (str): variable name or function name
+
+    Returns:
+        variable or function with the name of item_name
+
+    """
+    # override built_in module with debugtalk.py module
+    from httprunner import loader
+    built_in_module = loader.load_builtin_module()
+
+    if item_type == "variables":
+        try:
+            return built_in_module["variables"][item_name]
+        except KeyError:
+            raise exceptions.VariableNotFound("{} is not found.".format(item_name))
+    else:
+        # item_type == "functions":
+        try:
+            return built_in_module["functions"][item_name]
+        except KeyError:
+            raise exceptions.FunctionNotFound("{} is not found.".format(item_name))
+
+
 def get_mapping_variable(variable_name, variables_mapping):
     """ get variable from variables_mapping.
 
@@ -339,10 +368,10 @@ def get_mapping_variable(variable_name, variables_mapping):
         exceptions.VariableNotFound: variable is not found.
 
     """
-    try:
+    if variable_name in variables_mapping:
         return variables_mapping[variable_name]
-    except KeyError:
+    else:
-        raise exceptions.VariableNotFound("{} is not found.".format(variable_name))
+        return get_builtin_item("variables", variable_name)
 
 
 def get_mapping_function(function_name, functions_mapping):
@@ -363,6 +392,11 @@ def get_mapping_function(function_name, functions_mapping):
     if function_name in functions_mapping:
         return functions_mapping[function_name]
 
+    try:
+        return get_builtin_item("functions", function_name)
+    except exceptions.FunctionNotFound:
+        pass
+
     try:
         # check if builtin functions
         item_func = eval(function_name)
@@ -522,3 +556,118 @@ def parse_data(content, variables_mapping=None, functions_mapping=None):
         content = parse_string_variables(content, variables_mapping)
 
     return content
+
+
+def parse_tests(testcases, variables_mapping=None):
+    """ parse testcases configs, including variables/parameters/name/request.
+
+    Args:
+        testcases (list): testcase list, with config unparsed.
+            [
+                {   # testcase data structure
+                    "config": {
+                        "name": "desc1",
+                        "path": "testcase1_path",
+                        "variables": [],         # optional
+                        "request": {}            # optional
+                        "refs": {
+                            "debugtalk": {
+                                "variables": {},
+                                "functions": {}
+                            },
+                            "env": {},
+                            "def-api": {},
+                            "def-testcase": {}
+                        }
+                    },
+                    "teststeps": [
+                        # teststep data structure
+                        {
+                            'name': 'test step desc2',
+                            'variables': [],    # optional
+                            'extract': [],      # optional
+                            'validate': [],
+                            'request': {},
+                            'function_meta': {}
+                        },
+                        teststep2   # another teststep dict
+                    ]
+                },
+                testcase_dict_2     # another testcase dict
+            ]
+        variables_mapping (dict): if variables_mapping is specified, it will override variables in config block.
+
+    Returns:
+        list: parsed testcases list, with config variables/parameters/name/request parsed.
+
+    """
+    # exception_stage = "parse tests"
+    variables_mapping = variables_mapping or {}
+    parsed_testcases_list = []
+
+    for testcase in testcases:
+        testcase_config = testcase.setdefault("config", {})
+        project_mapping = testcase_config.pop(
+            "refs",
+            {
+                "debugtalk": {
+                    "variables": {},
+                    "functions": {}
+                },
+                "env": {},
+                "def-api": {},
+                "def-testcase": {}
+            }
+        )
+
+        # parse config parameters
+        config_parameters = testcase_config.pop("parameters", [])
+        cartesian_product_parameters_list = parse_parameters(
+            config_parameters,
+            project_mapping["debugtalk"]["variables"],
+            project_mapping["debugtalk"]["functions"]
+        ) or [{}]
+
+        for parameter_mapping in cartesian_product_parameters_list:
+            testcase_dict = copy.deepcopy(testcase)
+            config = testcase_dict.get("config")
+
+            # parse config variables
+            raw_config_variables = config.get("variables", [])
+            parsed_config_variables = parse_data(
+                raw_config_variables,
+                project_mapping["debugtalk"]["variables"],
+                project_mapping["debugtalk"]["functions"]
+            )
+
+            # priority: passed in > debugtalk.py > parameters > variables
+            # override variables mapping with parameters mapping
+            config_variables = utils.override_mapping_list(
+                parsed_config_variables, parameter_mapping)
+            # merge debugtalk.py module variables
+            config_variables.update(project_mapping["debugtalk"]["variables"])
+            # override variables mapping with passed in variables_mapping
+            config_variables = utils.override_mapping_list(
+                config_variables, variables_mapping)
+
+            testcase_dict["config"]["variables"] = config_variables
+
+            # parse config name
+            testcase_dict["config"]["name"] = parse_data(
+                testcase_dict["config"].get("name", ""),
+                config_variables,
+                project_mapping["debugtalk"]["functions"]
+            )
+
+            # parse config request
+            testcase_dict["config"]["request"] = parse_data(
+                testcase_dict["config"].get("request", {}),
+                config_variables,
+                project_mapping["debugtalk"]["functions"]
+            )
+
+            # put loaded project functions to config
+            testcase_dict["config"]["functions"] = project_mapping["debugtalk"]["functions"]
+            parsed_testcases_list.append(testcase_dict)
+
+    return parsed_testcases_list

httprunner/report.py

@@ -95,7 +95,7 @@ def render_html_report(summary, html_report_name=None, html_report_template=None
     logger.log_info("Start to render Html report ...")
     logger.log_debug("render data: {}".format(summary))
 
-    report_dir_path = os.path.join(loader.project_working_directory, "reports")
+    report_dir_path = os.path.join(os.getcwd(), "reports")
     start_at_timestamp = int(summary["time"]["start_at"])
     summary["time"]["start_datetime"] = datetime.fromtimestamp(start_at_timestamp).strftime('%Y-%m-%d %H:%M:%S')
     if html_report_name:

httprunner/templates/locustfile_template

@@ -2,16 +2,17 @@
 import zmq
 from locust import HttpLocust, TaskSet, task
 from httprunner import LocustRunner
+from httprunner.loader import load_tests
 
 
 class WebPageTasks(TaskSet):
     def on_start(self):
         self.test_runner = LocustRunner(self.client)
-        self.file_path = self.locust.file_path
+        self.testcases = loader.load_tests(self.locust.file_path)
 
     @task
     def test_specified_scenario(self):
-        self.test_runner.run(self.file_path)
+        self.test_runner.run(self.testcases)
 
 
 class WebPageUser(HttpLocust):

httprunner/validator.py

@@ -1,6 +1,8 @@
 # encoding: utf-8
+import os
 import types
 
+
 """ validate data format
 TODO: refactor with JSON schema validate
 """
@@ -46,6 +48,7 @@ def is_testcase(data_structure):
 
     return True
 
+
 def is_testcases(data_structure):
     """ check if data_structure is testcase or testcases list.
 
@@ -72,6 +75,31 @@ def is_testcases(data_structure):
     return True
 
 
+def is_testcase_path(path):
+    """ check if path is testcase path or path list.
+
+    Args:
+        path (str/list): file path or file path list.
+
+    Returns:
+        bool: True if path is valid file path or path list, otherwise False.
+
+    """
+    if not isinstance(path, (str, list)):
+        return False
+
+    if isinstance(path, list):
+        for p in path:
+            if not is_testcase_path(p):
+                return False
+
+    if isinstance(path, str):
+        if not os.path.exists(path):
+            return False
+
+    return True
+
+
 ###############################################################################
 ##   validate varibles and functions
 ###############################################################################
@@ -101,4 +129,3 @@ def is_variable(tup):
         return False
 
     return True
-

tests/test_api.py

@@ -2,7 +2,7 @@ import os
 import shutil
 import time
 
-from httprunner import HttpRunner, LocustRunner, loader
+from httprunner import HttpRunner, LocustRunner, api, loader, parser
 from locust import HttpLocust
 from tests.api_server import HTTPBIN_SERVER
 from tests.base import ApiServerUnittest
@@ -11,25 +11,22 @@ from tests.base import ApiServerUnittest
 class TestHttpRunner(ApiServerUnittest):
 
     def setUp(self):
-        self.testset_path = "tests/data/demo_testset_cli.yml"
+        self.testcase_cli_path = "tests/data/demo_testset_cli.yml"
         self.testcase_file_path_list = [
             os.path.join(
                 os.getcwd(), 'tests/data/demo_testset_hardcode.yml'),
             os.path.join(
                 os.getcwd(), 'tests/data/demo_testset_hardcode.json')
         ]
-        self.testcase = {
+        self.testcases = [{
-            'name': 'testset description',
             'config': {
-                'name': 'testset description',
+                'name': 'testcase description',
                 'request': {
                     'base_url': '',
                     'headers': {'User-Agent': 'python-requests/2.18.4'}
                 },
-                'variables': [],
+                'variables': []
-                'output': ['token']
             },
-            'api': {},
             'teststeps': [
                 {
                     'name': '/api/get-token',
@@ -63,7 +60,7 @@ class TestHttpRunner(ApiServerUnittest):
                     ]
                 }
             ]
-        }
+        }]
         self.reset_all()
 
     def reset_all(self):
@@ -72,46 +69,35 @@ class TestHttpRunner(ApiServerUnittest):
         return self.api_client.get(url, headers=headers)
 
     def test_text_run_times(self):
-        runner = HttpRunner().run(self.testset_path)
+        runner = HttpRunner().run(self.testcase_cli_path)
         self.assertEqual(runner.summary["stat"]["testsRun"], 10)
 
     def test_text_skip(self):
-        runner = HttpRunner().run(self.testset_path)
+        runner = HttpRunner().run(self.testcase_cli_path)
         self.assertEqual(runner.summary["stat"]["skipped"], 4)
 
     def test_html_report(self):
-        kwargs = {}
+        runner = HttpRunner().run(self.testcase_cli_path)
-        output_folder_name = os.path.basename(os.path.splitext(self.testset_path)[0])
-        runner = HttpRunner().run(self.testset_path)
         summary = runner.summary
         self.assertEqual(summary["stat"]["testsRun"], 10)
         self.assertEqual(summary["stat"]["skipped"], 4)
 
+        output_folder_name = "demo"
         runner.gen_html_report(html_report_name=output_folder_name)
-        report_save_dir = os.path.join(loader.project_working_directory, 'reports', output_folder_name)
+        report_save_dir = os.path.join(os.getcwd(), 'reports', output_folder_name)
         self.assertGreater(len(os.listdir(report_save_dir)), 0)
         shutil.rmtree(report_save_dir)
 
     def test_run_testcases(self):
-        testcases = [self.testcase]
+        runner = HttpRunner().run(self.testcases)
-        runner = HttpRunner().run(testcases)
         summary = runner.summary
         self.assertTrue(summary["success"])
         self.assertEqual(summary["stat"]["testsRun"], 2)
         self.assertIn("details", summary)
         self.assertIn("records", summary["details"][0])
 
-    def test_run_testcase(self):
-        testcases = self.testcase
-        runner = HttpRunner().run(testcases)
-        summary = runner.summary
-        self.assertTrue(summary["success"])
-        self.assertEqual(summary["stat"]["testsRun"], 2)
-        self.assertIn("records", summary["details"][0])
-
     def test_run_yaml_upload(self):
-        testset_path = "tests/httpbin/upload.yml"
+        runner = HttpRunner().run("tests/httpbin/upload.yml")
-        runner = HttpRunner().run(testset_path)
         summary = runner.summary
         self.assertTrue(summary["success"])
         self.assertEqual(summary["stat"]["testsRun"], 1)
@@ -121,7 +107,14 @@ class TestHttpRunner(ApiServerUnittest):
     def test_run_post_data(self):
         testcases = [
             {
-                "name": "post data",
+                "config": {
+                    'name': "post data",
+                    'request': {
+                        'base_url': '',
+                        'headers': {'User-Agent': 'python-requests/2.18.4'}
+                    },
+                    'variables': []
+                },
                 "teststeps": [
                     {
                         "name": "post data",
@@ -137,7 +130,6 @@ class TestHttpRunner(ApiServerUnittest):
                             {"eq": ["status_code", 200]}
                         ]
                     }
-
                 ]
             }
         ]
@@ -148,18 +140,16 @@ class TestHttpRunner(ApiServerUnittest):
         self.assertEqual(summary["details"][0]["records"][0]["meta_data"]["response"]["json"]["data"], "abc")
 
     def test_html_report_repsonse_image(self):
-        testset_path = "tests/httpbin/load_image.yml"
+        runner = HttpRunner().run("tests/httpbin/load_image.yml")
-        runner = HttpRunner().run(testset_path)
         summary = runner.summary
-        output_folder_name = os.path.basename(os.path.splitext(testset_path)[0])
+        output_folder_name = "demo"
         report = runner.gen_html_report(html_report_name=output_folder_name)
         self.assertTrue(os.path.isfile(report))
-        report_save_dir = os.path.join(loader.project_working_directory, 'reports', output_folder_name)
+        report_save_dir = os.path.join(os.getcwd(), 'reports', output_folder_name)
         shutil.rmtree(report_save_dir)
 
     def test_testcase_layer(self):
-        testcase_path = "tests/testcases/smoketest.yml"
+        runner = HttpRunner(failfast=True).run("tests/testcases/smoketest.yml")
-        runner = HttpRunner(failfast=True).run(testcase_path)
         summary = runner.summary
         self.assertTrue(summary["success"])
         self.assertEqual(summary["stat"]["testsRun"], 8)
@@ -167,7 +157,6 @@ class TestHttpRunner(ApiServerUnittest):
     def test_run_httprunner_with_hooks(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/httpbin/hooks.yml')
-
         start_time = time.time()
         runner = HttpRunner().run(testcase_file_path)
         end_time = time.time()
@@ -180,7 +169,7 @@ class TestHttpRunner(ApiServerUnittest):
             {
                 "config": {
                     "name": "test teardown hooks",
-                    "path": "tests/httpbin/hooks.yml"
+                    "refs": loader.load_project_tests("tests")
                 },
                 "teststeps": [
                     {
@@ -214,9 +203,8 @@ class TestHttpRunner(ApiServerUnittest):
     def test_run_httprunner_with_teardown_hooks_not_exist_attribute(self):
         testcases = [
             {
-                "name": "test teardown hooks",
                 "config": {
-                    "path": "tests/httpbin/hooks.yml"
+                    "name": "test teardown hooks"
                 },
                 "teststeps": [
                     {
@@ -244,8 +232,9 @@ class TestHttpRunner(ApiServerUnittest):
     def test_run_httprunner_with_teardown_hooks_error(self):
         testcases = [
             {
-                "name": "test teardown hooks",
+                "config": {
-                "config": {},
+                    "name": "test teardown hooks"
+                },
                 "teststeps": [
                     {
                         "name": "test teardown hooks",
@@ -269,30 +258,34 @@ class TestHttpRunner(ApiServerUnittest):
     def test_run_testcase_hardcode(self):
         for testcase_file_path in self.testcase_file_path_list:
             runner = HttpRunner().run(testcase_file_path)
-            self.assertTrue(runner.summary["success"])
+            summary = runner.summary
+            self.assertTrue(summary["success"])
+            self.assertEqual(summary["stat"]["testsRun"], 3)
+            self.assertEqual(summary["stat"]["successes"], 3)
 
     def test_run_testcases_hardcode(self):
         runner = HttpRunner().run(self.testcase_file_path_list)
         summary = runner.summary
         self.assertTrue(summary["success"])
+        self.assertTrue(summary["success"])
         self.assertEqual(summary["stat"]["testsRun"], 6)
         self.assertEqual(summary["stat"]["successes"], 6)
 
-    def test_run_testset_template_variables(self):
+    def test_run_testcase_template_variables(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/data/demo_testset_variables.yml')
         runner = HttpRunner().run(testcase_file_path)
         summary = runner.summary
         self.assertTrue(summary["success"])
 
-    def test_run_testset_template_import_functions(self):
+    def test_run_testcase_template_import_functions(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/data/demo_testset_functions.yml')
         runner = HttpRunner().run(testcase_file_path)
         summary = runner.summary
         self.assertTrue(summary["success"])
 
-    def test_run_testset_layered(self):
+    def test_run_testcase_layered(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/data/demo_testset_layer.yml')
         runner = HttpRunner().run(testcase_file_path)
@@ -347,10 +340,10 @@ class TestHttpRunner(ApiServerUnittest):
     def test_run_testcase_with_parameters_name(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/data/demo_parameters.yml')
+        testcases = loader.load_tests(testcase_file_path)
+        parsed_testcases = parser.parse_tests(testcases)
         runner = HttpRunner()
-        testcases = runner.load_tests(testcase_file_path)
+        test_suite = runner._add_tests(parsed_testcases)
-        parsed_testcases = runner.parse_tests(testcases)
-        unittest_runner, test_suite = runner.initialize(parsed_testcases)
 
         self.assertEqual(
             test_suite._tests[0].teststeps[0]['name'],
@@ -377,41 +370,6 @@ class TestHttpRunner(ApiServerUnittest):
             'get token with iOS/10.3 and test2'
         )
 
-    def test_load_tests(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/demo_testcase.yml')
-        runner = HttpRunner()
-        testcases = runner.load_tests(testcase_file_path)
-        self.assertIsInstance(testcases, list)
-        self.assertEqual(
-            testcases[0]["config"]["request"],
-            '$demo_default_request'
-        )
-        self.assertEqual(testcases[0]["config"]["name"], '123$var_a')
-        self.assertIn(
-            "sum_two",
-            runner.project_mapping["debugtalk"]["functions"]
-        )
-
-    def test_parse_tests(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/demo_testcase.yml')
-        runner = HttpRunner()
-        testcases = runner.load_tests(testcase_file_path)
-        parsed_testcases = runner.parse_tests(testcases)
-        self.assertEqual(parsed_testcases[0]["config"]["variables"]["var_c"], 3)
-        self.assertEqual(len(parsed_testcases), 2 * 2)
-        self.assertEqual(
-            parsed_testcases[0]["config"]["request"]["base_url"],
-            '$BASE_URL'
-        )
-        self.assertEqual(
-            parsed_testcases[0]["config"]["variables"]["BASE_URL"],
-            'http://127.0.0.1:5000'
-        )
-        self.assertIsInstance(parsed_testcases, list)
-        self.assertEqual(parsed_testcases[0]["config"]["name"], '12311')
-
     def test_validate_response_content(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/httpbin/basic.yml')

tests/test_context.py

@@ -9,8 +9,8 @@ from tests.base import ApiServerUnittest
 class TestContext(ApiServerUnittest):
 
     def setUp(self):
-        loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
+        project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
-        self.debugtalk_module = loader.project_mapping["debugtalk"]
+        self.debugtalk_module = project_mapping["debugtalk"]
 
         self.context = context.Context(
             self.debugtalk_module["variables"],
@@ -22,7 +22,6 @@ class TestContext(ApiServerUnittest):
     def test_init_context_functions(self):
         context_functions = self.context.TESTCASE_SHARED_FUNCTIONS_MAPPING
         self.assertIn("gen_md5", context_functions)
-        self.assertIn("equals", context_functions)
 
     def test_init_context_variables(self):
         self.assertEqual(

tests/test_loader.py

@@ -133,29 +133,28 @@ class TestFileLoader(unittest.TestCase):
         self.assertEqual([], files)
 
     def test_load_dot_env_file(self):
-        loader.project_working_directory = os.path.join(
+        dot_env_path = os.path.join(
-            os.getcwd(), "tests",
+            os.getcwd(), "tests", ".env"
         )
-        env_variables_mapping = loader.load_dot_env_file()
+        env_variables_mapping = loader.load_dot_env_file(dot_env_path)
         self.assertIn("PROJECT_KEY", env_variables_mapping)
         self.assertEqual(env_variables_mapping["UserName"], "debugtalk")
 
     def test_load_custom_dot_env_file(self):
-        loader.project_working_directory = os.path.join(
+        dot_env_path = os.path.join(
-            os.getcwd(), "tests",
+            os.getcwd(), "tests", "data", "test.env"
         )
-        loader.dot_env_path = "tests/data/test.env"
+        env_variables_mapping = loader.load_dot_env_file(dot_env_path)
-        env_variables_mapping = loader.load_dot_env_file()
         self.assertIn("PROJECT_KEY", env_variables_mapping)
         self.assertEqual(env_variables_mapping["UserName"], "test")
         self.assertEqual(env_variables_mapping["content_type"], "application/json; charset=UTF-8")
-        loader.dot_env_path = None
 
     def test_load_env_path_not_exist(self):
-        loader.project_working_directory = os.path.join(
+        dot_env_path = os.path.join(
             os.getcwd(), "tests", "data",
         )
-        loader.load_dot_env_file()
+        with self.assertRaises(exceptions.FileNotFound):
+            loader.load_dot_env_file(dot_env_path)
 
     def test_locate_file(self):
         with self.assertRaises(exceptions.FileNotFound):
@@ -198,14 +197,13 @@ class TestModuleLoader(unittest.TestCase):
         self.assertNotIn("is_py3", functions_dict)
 
     def test_load_debugtalk_module(self):
-        loader.load_project_tests(os.path.join(os.getcwd(), "httprunner"))
+        project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "httprunner"))
-        imported_module_items = loader.project_mapping["debugtalk"]
+        imported_module_items = project_mapping["debugtalk"]
-        self.assertIn("equals", imported_module_items["functions"])
         self.assertNotIn("SECRET_KEY", imported_module_items["variables"])
         self.assertNotIn("alter_response", imported_module_items["functions"])
 
-        loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
+        project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
-        imported_module_items = loader.project_mapping["debugtalk"]
+        imported_module_items = project_mapping["debugtalk"]
         self.assertEqual(
             imported_module_items["variables"]["SECRET_KEY"],
             "DebugTalk"
@@ -229,6 +227,11 @@ class TestModuleLoader(unittest.TestCase):
             loader.get_module_item(module_mapping, "functions", "gen_md4")
 
     def test_get_module_item_variables(self):
+        dot_env_path = os.path.join(
+            os.getcwd(), "tests", ".env"
+        )
+        loader.load_dot_env_file(dot_env_path)
+
         from tests import debugtalk
         module_mapping = loader.load_python_module(debugtalk)
 
@@ -258,15 +261,30 @@ class TestModuleLoader(unittest.TestCase):
             None
         )
 
+    def test_load_tests(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_testcase.yml')
+        testcases = loader.load_tests(testcase_file_path)
+        self.assertIsInstance(testcases, list)
+        self.assertEqual(
+            testcases[0]["config"]["request"],
+            '$demo_default_request'
+        )
+        self.assertEqual(testcases[0]["config"]["name"], '123$var_a')
+        self.assertIn(
+            "sum_two",
+            testcases[0]["config"]["refs"]["debugtalk"]["functions"]
+        )
+
 
 class TestSuiteLoader(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
+        cls.project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
 
     def test_load_test_file_testcase(self):
-        testcase = loader._load_test_file("tests/testcases/smoketest.yml")
+        testcase = loader._load_test_file("tests/testcases/smoketest.yml", self.project_mapping)
         self.assertEqual(testcase["config"]["name"], "smoketest")
         self.assertIn("device_sn", testcase["config"]["variables"][0])
         self.assertEqual(len(testcase["teststeps"]), 8)
@@ -274,7 +292,7 @@ class TestSuiteLoader(unittest.TestCase):
 
     def test_get_block_by_name(self):
         ref_call = "get_user($uid, $token)"
-        block = loader._get_block_by_name(ref_call, "def-api")
+        block = loader._get_block_by_name(ref_call, "def-api", self.project_mapping)
         self.assertEqual(block["request"]["url"], "/api/users/$uid")
         self.assertEqual(block["function_meta"]["func_name"], "get_user")
         self.assertEqual(block["function_meta"]["args"], ['$uid', '$token'])
@@ -282,11 +300,14 @@ class TestSuiteLoader(unittest.TestCase):
     def test_get_block_by_name_args_mismatch(self):
         ref_call = "get_user($uid, $token, $var)"
         with self.assertRaises(exceptions.ParamsError):
-            loader._get_block_by_name(ref_call, "def-api")
+            loader._get_block_by_name(ref_call, "def-api", self.project_mapping)
 
     def test_override_block(self):
         def_block = loader._get_block_by_name(
-            "get_token($user_agent, $device_sn, $os_platform, $app_version)", "def-api")
+            "get_token($user_agent, $device_sn, $os_platform, $app_version)",
+            "def-api",
+            self.project_mapping
+        )
         test_block = {
             "name": "override block",
             "variables": [
@@ -306,20 +327,20 @@ class TestSuiteLoader(unittest.TestCase):
         self.assertIn({'check': 'content.token', 'comparator': 'len_eq', 'expect': 32}, test_block["validate"])
 
     def test_get_test_definition_api(self):
-        api_def = loader._get_test_definition("get_headers", "def-api")
+        api_def = loader._get_test_definition("get_headers", "def-api", self.project_mapping)
         self.assertEqual(api_def["request"]["url"], "/headers")
         self.assertEqual(len(api_def["setup_hooks"]), 2)
         self.assertEqual(len(api_def["teardown_hooks"]), 1)
 
         with self.assertRaises(exceptions.ApiNotFound):
-            loader._get_test_definition("get_token_XXX", "def-api")
+            loader._get_test_definition("get_token_XXX", "def-api", self.project_mapping)
 
     def test_get_test_definition_suite(self):
-        api_def = loader._get_test_definition("create_and_check", "def-testcase")
+        api_def = loader._get_test_definition("create_and_check", "def-testcase", self.project_mapping)
         self.assertEqual(api_def["config"]["name"], "create user and check result.")
 
         with self.assertRaises(exceptions.TestcaseNotFound):
-            loader._get_test_definition("create_and_check_XXX", "def-testcase")
+            loader._get_test_definition("create_and_check_XXX", "def-testcase", self.project_mapping)
 
     def test_merge_validator(self):
         def_validators = [
@@ -379,51 +400,59 @@ class TestSuiteLoader(unittest.TestCase):
         )
 
     def test_load_testcases_by_path_files(self):
-        testsets_list = []
+        testcases_list = []
 
         # absolute file path
         path = os.path.join(
             os.getcwd(), 'tests/data/demo_testset_hardcode.json')
-        testset_list = loader.load_testcases(path)
+        testcases_list = loader.load_tests(path)
-        self.assertEqual(len(testset_list), 1)
+        self.assertEqual(len(testcases_list), 1)
-        self.assertEqual(len(testset_list[0]["teststeps"]), 3)
+        self.assertEqual(len(testcases_list[0]["teststeps"]), 3)
-        testsets_list.extend(testset_list)
+        self.assertEqual(
+            testcases_list[0]["config"]["refs"]["debugtalk"]["variables"]["SECRET_KEY"],
+            "DebugTalk"
+        )
+        self.assertIn("get_sign", testcases_list[0]["config"]["refs"]["debugtalk"]["functions"])
 
         # relative file path
         path = 'tests/data/demo_testset_hardcode.yml'
-        testset_list = loader.load_testcases(path)
+        testcases_list = loader.load_tests(path)
-        self.assertEqual(len(testset_list), 1)
+        self.assertEqual(len(testcases_list), 1)
-        self.assertEqual(len(testset_list[0]["teststeps"]), 3)
+        self.assertEqual(len(testcases_list[0]["teststeps"]), 3)
-        testsets_list.extend(testset_list)
+        self.assertEqual(
+            testcases_list[0]["config"]["refs"]["debugtalk"]["variables"]["SECRET_KEY"],
+            "DebugTalk"
+        )
+        self.assertIn("get_sign", testcases_list[0]["config"]["refs"]["debugtalk"]["functions"])
 
         # list/set container with file(s)
         path = [
             os.path.join(os.getcwd(), 'tests/data/demo_testset_hardcode.json'),
             'tests/data/demo_testset_hardcode.yml'
         ]
-        testset_list = loader.load_testcases(path)
+        testcases_list = loader.load_tests(path)
-        self.assertEqual(len(testset_list), 2)
+        self.assertEqual(len(testcases_list), 2)
-        self.assertEqual(len(testset_list[0]["teststeps"]), 3)
+        self.assertEqual(len(testcases_list[0]["teststeps"]), 3)
-        self.assertEqual(len(testset_list[1]["teststeps"]), 3)
+        self.assertEqual(len(testcases_list[1]["teststeps"]), 3)
-        testsets_list.extend(testset_list)
+        testcases_list.extend(testcases_list)
-        self.assertEqual(len(testsets_list), 4)
+        self.assertEqual(len(testcases_list), 4)
 
-        for testset in testsets_list:
+        for testcase in testcases_list:
-            for test in testset["teststeps"]:
+            for teststep in testcase["teststeps"]:
-                self.assertIn('name', test)
+                self.assertIn('name', teststep)
-                self.assertIn('request', test)
+                self.assertIn('request', teststep)
-                self.assertIn('url', test['request'])
+                self.assertIn('url', teststep['request'])
-                self.assertIn('method', test['request'])
+                self.assertIn('method', teststep['request'])
 
     def test_load_testcases_by_path_folder(self):
         # absolute folder path
         path = os.path.join(os.getcwd(), 'tests/data')
-        testset_list_1 = loader.load_testcases(path)
+        testset_list_1 = loader.load_tests(path)
         self.assertGreater(len(testset_list_1), 4)
 
         # relative folder path
         path = 'tests/data/'
-        testset_list_2 = loader.load_testcases(path)
+        testset_list_2 = loader.load_tests(path)
         self.assertEqual(len(testset_list_1), len(testset_list_2))
 
         # list/set container with file(s)
@@ -431,19 +460,19 @@ class TestSuiteLoader(unittest.TestCase):
             os.path.join(os.getcwd(), 'tests/data'),
             'tests/data/'
         ]
-        testset_list_3 = loader.load_testcases(path)
+        testset_list_3 = loader.load_tests(path)
         self.assertEqual(len(testset_list_3), 2 * len(testset_list_1))
 
     def test_load_testcases_by_path_not_exist(self):
         # absolute folder path
         path = os.path.join(os.getcwd(), 'tests/data_not_exist')
         with self.assertRaises(exceptions.FileNotFound):
-            loader.load_testcases(path)
+            loader.load_tests(path)
 
         # relative folder path
         path = 'tests/data_not_exist'
         with self.assertRaises(exceptions.FileNotFound):
-            loader.load_testcases(path)
+            loader.load_tests(path)
 
         # list/set container with file(s)
         path = [
@@ -451,17 +480,17 @@ class TestSuiteLoader(unittest.TestCase):
             'tests/data_not_exist/'
         ]
         with self.assertRaises(exceptions.FileNotFound):
-            loader.load_testcases(path)
+            loader.load_tests(path)
 
     def test_load_testcases_by_path_layered(self):
         path = os.path.join(
             os.getcwd(), 'tests/data/demo_testset_layer.yml')
-        testsets_list = loader.load_testcases(path)
+        testcases_list = loader.load_tests(path)
-        self.assertIn("variables", testsets_list[0]["config"])
+        self.assertIn("variables", testcases_list[0]["config"])
-        self.assertIn("request", testsets_list[0]["config"])
+        self.assertIn("request", testcases_list[0]["config"])
-        self.assertIn("request", testsets_list[0]["teststeps"][0])
+        self.assertIn("request", testcases_list[0]["teststeps"][0])
-        self.assertIn("url", testsets_list[0]["teststeps"][0]["request"])
+        self.assertIn("url", testcases_list[0]["teststeps"][0]["request"])
-        self.assertIn("validate", testsets_list[0]["teststeps"][0])
+        self.assertIn("validate", testcases_list[0]["teststeps"][0])
 
     def test_load_folder_content(self):
         path = os.path.join(os.getcwd(), "tests", "api")
@@ -507,8 +536,7 @@ class TestSuiteLoader(unittest.TestCase):
         )
 
     def test_load_project_tests(self):
-        loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
+        project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
-        project_mapping = loader.project_mapping
         self.assertEqual(project_mapping["debugtalk"]["variables"]["SECRET_KEY"], "DebugTalk")
         self.assertIn("get_token", project_mapping["def-api"])
         self.assertIn("setup_and_reset", project_mapping["def-testcase"])

tests/test_parser.py

@@ -384,6 +384,10 @@ class TestParser(unittest.TestCase):
             os.getcwd(),
             "tests/data/demo_parameters.yml"
         )
+        dot_env_path = os.path.join(
+            os.getcwd(), "tests", ".env"
+        )
+        loader.load_dot_env_file(dot_env_path)
         from tests import debugtalk
         debugtalk_module = loader.load_python_module(debugtalk)
         cartesian_product_parameters = parser.parse_parameters(
@@ -412,8 +416,7 @@ class TestParser(unittest.TestCase):
         )
 
     def test_parse_parameters_mix(self):
-        loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
+        project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
-        project_mapping = loader.project_mapping
 
         parameters = [
             {"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]},
@@ -432,3 +435,21 @@ class TestParser(unittest.TestCase):
             len(cartesian_product_parameters),
             3 * 2 * 3
         )
+
+    def test_parse_tests(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_testcase.yml')
+        testcases = loader.load_tests(testcase_file_path)
+        parsed_testcases = parser.parse_tests(testcases)
+        self.assertEqual(parsed_testcases[0]["config"]["variables"]["var_c"], 3)
+        self.assertEqual(len(parsed_testcases), 2 * 2)
+        self.assertEqual(
+            parsed_testcases[0]["config"]["request"]["base_url"],
+            '$BASE_URL'
+        )
+        self.assertEqual(
+            parsed_testcases[0]["config"]["variables"]["BASE_URL"],
+            'http://127.0.0.1:5000'
+        )
+        self.assertIsInstance(parsed_testcases, list)
+        self.assertEqual(parsed_testcases[0]["config"]["name"], '12311')

tests/test_runner.py

@@ -10,8 +10,8 @@ from tests.base import ApiServerUnittest
 class TestRunner(ApiServerUnittest):
 
     def setUp(self):
-        loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
+        project_mapping = loader.load_project_tests(os.path.join(os.getcwd(), "tests"))
-        self.debugtalk_module = loader.project_mapping["debugtalk"]
+        self.debugtalk_module = project_mapping["debugtalk"]
         config_dict = {
             "variables": self.debugtalk_module["variables"],
             "functions": self.debugtalk_module["functions"]
@@ -226,7 +226,7 @@ class TestRunner(ApiServerUnittest):
     def test_run_testcase_with_empty_header(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/data/test_bugfix.yml')
-        testsets = loader.load_testcases(testcase_file_path)
+        testsets = loader.load_tests(testcase_file_path)
         testset = testsets[0]
         config_dict_headers = testset["config"]["request"]["headers"]
         test_dict_headers = testset["teststeps"][0]["request"]["headers"]