feat: make testsuite and run testsuite

docs/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## 3.0.4 (2020-05-18)
 
+**Added**
+
+- feat: make testsuite and run testsuite
+
 **Fixed**
 
 - fix: extract response cookies

examples/postman_echo/request_methods/demo_testsuite.yml

@@ -0,0 +1,15 @@
+config:
+    name: "demo testsuite"
+#    variables: ${get_variable()}
+
+testcases:
+-
+    name: request with functions
+    testcase: request_methods/request_with_functions.yml
+    variables:
+        var1: testsuite_val1
+-
+    name: request with referenced testcase
+    testcase: request_methods/request_with_testcase_reference.yml
+    variables:
+        var2: testsuite_val2

examples/postman_echo/request_methods/demo_testsuite/request_with_functions_test.py

@@ -0,0 +1,89 @@
+# NOTICE: Generated By HttpRunner. DO'NOT EDIT!
+# FROM: examples/postman_echo/request_methods/demo_testsuite/request_with_functions.yml
+from httprunner import HttpRunner, TConfig, TStep
+
+
+class TestCaseRequestWithFunctions(HttpRunner):
+    config = TConfig(
+        **{
+            "name": "request with functions",
+            "variables": {"foo1": "session_bar1", "var1": "testsuite_val1"},
+            "base_url": "https://postman-echo.com",
+            "verify": False,
+            "path": "examples/postman_echo/request_methods/demo_testsuite/request_with_functions_test.py",
+        }
+    )
+
+    teststeps = [
+        TStep(
+            **{
+                "name": "get with params",
+                "variables": {
+                    "foo1": "bar1",
+                    "foo2": "session_bar2",
+                    "sum_v": "${sum_two(1, 2)}",
+                },
+                "request": {
+                    "method": "GET",
+                    "url": "/get",
+                    "params": {"foo1": "$foo1", "foo2": "$foo2", "sum_v": "$sum_v"},
+                    "headers": {"User-Agent": "HttpRunner/${get_httprunner_version()}"},
+                },
+                "extract": {"session_foo2": "body.args.foo2"},
+                "validate": [
+                    {"eq": ["status_code", 200]},
+                    {"eq": ["body.args.foo1", "session_bar1"]},
+                    {"eq": ["body.args.sum_v", 3]},
+                    {"eq": ["body.args.foo2", "session_bar2"]},
+                ],
+            }
+        ),
+        TStep(
+            **{
+                "name": "post raw text",
+                "variables": {"foo1": "hello world", "foo3": "$session_foo2"},
+                "request": {
+                    "method": "POST",
+                    "url": "/post",
+                    "headers": {
+                        "User-Agent": "HttpRunner/${get_httprunner_version()}",
+                        "Content-Type": "text/plain",
+                    },
+                    "data": "This is expected to be sent back as part of response body: $foo1-$foo3.",
+                },
+                "validate": [
+                    {"eq": ["status_code", 200]},
+                    {
+                        "eq": [
+                            "body.data",
+                            "This is expected to be sent back as part of response body: session_bar1-session_bar2.",
+                        ]
+                    },
+                ],
+            }
+        ),
+        TStep(
+            **{
+                "name": "post form data",
+                "variables": {"foo1": "bar1", "foo2": "bar2"},
+                "request": {
+                    "method": "POST",
+                    "url": "/post",
+                    "headers": {
+                        "User-Agent": "HttpRunner/${get_httprunner_version()}",
+                        "Content-Type": "application/x-www-form-urlencoded",
+                    },
+                    "data": "foo1=$foo1&foo2=$foo2",
+                },
+                "validate": [
+                    {"eq": ["status_code", 200]},
+                    {"eq": ["body.form.foo1", "session_bar1"]},
+                    {"eq": ["body.form.foo2", "bar2"]},
+                ],
+            }
+        ),
+    ]
+
+
+if __name__ == "__main__":
+    TestCaseRequestWithFunctions().test_start()

examples/postman_echo/request_methods/demo_testsuite/request_with_testcase_reference_test.py

@@ -0,0 +1,29 @@
+# NOTICE: Generated By HttpRunner. DO'NOT EDIT!
+# FROM: examples/postman_echo/request_methods/demo_testsuite/request_with_testcase_reference.yml
+from httprunner import HttpRunner, TConfig, TStep
+
+
+class TestCaseRequestWithTestcaseReference(HttpRunner):
+    config = TConfig(
+        **{
+            "name": "request with referenced testcase",
+            "variables": {"foo1": "session_bar1", "var2": "testsuite_val2"},
+            "base_url": "https://postman-echo.com",
+            "verify": False,
+            "path": "examples/postman_echo/request_methods/demo_testsuite/request_with_testcase_reference_test.py",
+        }
+    )
+
+    teststeps = [
+        TStep(
+            **{
+                "name": "request with variables",
+                "variables": {"foo1": "override_bar1"},
+                "testcase": "request_methods/request_with_variables.yml",
+            }
+        ),
+    ]
+
+
+if __name__ == "__main__":
+    TestCaseRequestWithTestcaseReference().test_start()

httprunner/cli.py

@@ -20,15 +20,14 @@ def init_parser_run(subparsers):
 
 def main_run(extra_args):
     tests_path_list = []
-    for index, item in enumerate(extra_args):
+    extra_args_new = []
+    for item in extra_args:
         if not os.path.exists(item):
             # item is not file/folder path
-            continue
+            extra_args_new.append(item)
-        elif os.path.isfile(item):
+        else:
-            # replace YAML/JSON file path with generated python file
+            # item is file/folder path
-            extra_args[index], _ = convert_testcase_path(item)
+            tests_path_list.append(item)
-
-        tests_path_list.append(item)
 
     if len(tests_path_list) == 0:
         # has not specified any testcase path
@@ -39,9 +38,11 @@ def main_run(extra_args):
         logger.error("No valid testcases found, exit 1.")
         sys.exit(1)
 
-    if "-s" not in extra_args:
+    extra_args_new.extend(testcase_path_list)
-        extra_args.insert(0, "-s")
+    if "-s" not in extra_args_new:
-    pytest.main(extra_args)
+        extra_args_new.insert(0, "-s")
+
+    pytest.main(extra_args_new)
 
 
 def main():

httprunner/exceptions.py

@@ -44,6 +44,10 @@ class TestCaseFormatError(MyBaseError):
     pass
 
 
+class TestSuiteFormatError(MyBaseError):
+    pass
+
+
 class ParamsError(MyBaseError):
     pass
 

httprunner/ext/make/__init__.py

@@ -6,12 +6,12 @@ import jinja2
 from loguru import logger
 
 from httprunner import exceptions
-from httprunner.exceptions import TestCaseFormatError
 from httprunner.loader import (
-    load_testcase_file,
     load_folder_files,
     load_test_file,
     load_testcase,
+    load_testsuite,
+    get_project_working_directory,
 )
 
 __TMPL__ = """# NOTICE: Generated By HttpRunner. DO'NOT EDIT!
@@ -44,7 +44,9 @@ def convert_testcase_path(testcase_path: Text) -> Tuple[Text, Text]:
 
     file_suffix = file_suffix.lower()
     if file_suffix not in [".json", ".yml", ".yaml"]:
-        raise exceptions.ParamsError("")
+        raise exceptions.ParamsError(
+            "testcase file should have .yaml/.yml/.json suffix"
+        )
 
     file_name = raw_file_name.replace(" ", "_").replace(".", "_").replace("-", "_")
     testcase_dir = os.path.dirname(testcase_path)
@@ -56,7 +58,23 @@ def convert_testcase_path(testcase_path: Text) -> Tuple[Text, Text]:
     return testcase_python_path, name_in_title_case
 
 
+def format_pytest_with_black(python_path: Text):
+    logger.info(f"format pytest case with black: {python_path}")
+    try:
+        subprocess.run(["black", python_path])
+    except subprocess.CalledProcessError as ex:
+        logger.error(ex)
+
+
 def make_testcase(testcase: Dict) -> Union[str, None]:
+    """convert valid testcase dict to pytest file path"""
+    try:
+        # validate testcase format
+        load_testcase(testcase)
+    except exceptions.TestCaseFormatError as ex:
+        logger.error(f"TestCaseFormatError: {ex}")
+        raise
+
     testcase_path = testcase["config"]["path"]
     logger.info(f"start to make testcase: {testcase_path}")
 
@@ -74,21 +92,63 @@ def make_testcase(testcase: Dict) -> Union[str, None]:
     }
     content = template.render(data)
 
+    os.makedirs(os.path.dirname(testcase_python_path), exist_ok=True)
     with open(testcase_python_path, "w") as f:
         f.write(content)
 
     logger.info(f"generated testcase: {testcase_python_path}")
+    format_pytest_with_black(testcase_python_path)
     return testcase_python_path
 
 
-def format_with_black(tests_path: Text):
+def make_testsuite(testsuite: Dict) -> List[Text]:
-    logger.info("format testcases with black ...")
+    """convert valid testsuite dict to pytest folder with testcases"""
-    tests_path, _ = convert_testcase_path(tests_path)
-
     try:
-        subprocess.run(["black", tests_path])
+        # validate testcase format
-    except subprocess.CalledProcessError as ex:
+        load_testsuite(testsuite)
-        logger.error(ex)
+    except exceptions.TestSuiteFormatError as ex:
+        logger.error(f"TestSuiteFormatError: {ex}")
+        raise
+
+    config = testsuite["config"]
+    testsuite_path = config["path"]
+    logger.info(f"start to make testsuite: {testsuite_path}")
+
+    testcase_files = []
+    project_working_directory = get_project_working_directory(testsuite_path)
+
+    for testcase in testsuite["testcases"]:
+        # get referenced testcase content
+        testcase_file = testcase["testcase"]
+        testcase_path = os.path.join(project_working_directory, testcase_file)
+        testcase_dict = load_test_file(testcase_path)
+        testcase_dict.setdefault("config", {})
+
+        # override testcase name
+        testcase_dict["config"]["name"] = testcase["name"]
+        # override base_url
+        base_url = testsuite["config"].get("base_url") or testcase.get("base_url")
+        if base_url:
+            testcase_dict["config"]["base_url"] = base_url
+        # override variables
+        testcase_dict["config"].setdefault("variables", {})
+        testcase_dict["config"]["variables"].update(
+            testcase.get("variables", {})
+        )
+        testcase_dict["config"]["variables"].update(
+            testsuite["config"].get("variables", {})
+        )
+
+        # create directory with testsuite file name, put its testcases under this directory
+        testcase_dict["config"]["path"] = os.path.join(
+            os.path.splitext(testsuite_path)[0], os.path.basename(testcase_path)
+        )
+
+        # make testcase
+        testcase_path = make_testcase(testcase_dict)
+        testcase_files.append(testcase_path)
+
+    return testcase_files
 
 
 def __make(tests_path: Text) -> List:
@@ -110,19 +170,25 @@ def __make(tests_path: Text) -> List:
             logger.warning(ex)
             continue
 
+        # testcase
         if "teststeps" in test_content:
-            # testcase
             try:
-                # validate testcase format
+                testcase_file = make_testcase(test_content)
-                load_testcase(test_content)
             except exceptions.TestCaseFormatError:
                 continue
 
-            testcase_file = make_testcase(test_content)
             testcase_path_list.append(testcase_file)
+
+        # testsuite
         elif "testcases" in test_content:
-            # testsuite
+            try:
-            pass
+                testcase_files = make_testsuite(test_content)
+            except exceptions.TestSuiteFormatError:
+                continue
+
+            testcase_path_list.extend(testcase_files)
+
+        # invalid format
         else:
             raise exceptions.FileFormatError(
                 f"test file is neither testcase nor testsuite: {test_file}"
@@ -132,7 +198,6 @@ def __make(tests_path: Text) -> List:
         logger.warning(f"No valid testcase generated on {tests_path}")
         return []
 
-    format_with_black(tests_path)
     return testcase_path_list
 
 

httprunner/ext/make/make_test.py

@@ -1,5 +1,6 @@
 import unittest
-from httprunner.ext.make import make_testcase, main_make, convert_testcase_path
+
+from httprunner.ext.make import main_make, convert_testcase_path
 
 
 class TestLoader(unittest.TestCase):
@@ -52,3 +53,16 @@ class TestLoader(unittest.TestCase):
             "/path/to 2/幕布login_test.py",
         )
         self.assertEqual(convert_testcase_path("/path/to/幕布login.yml")[1], "幕布Login")
+
+    def test_make_testsuite(self):
+        path = ["examples/postman_echo/request_methods/demo_testsuite.yml"]
+        testcase_python_list = main_make(path)
+        self.assertEqual(len(testcase_python_list), 2)
+        self.assertIn(
+            "examples/postman_echo/request_methods/demo_testsuite/request_with_functions_test.py",
+            testcase_python_list,
+        )
+        self.assertIn(
+            "examples/postman_echo/request_methods/demo_testsuite/request_with_testcase_reference_test.py",
+            testcase_python_list,
+        )

httprunner/loader.py

@@ -13,7 +13,7 @@ from pydantic import ValidationError
 
 from httprunner import builtin, utils
 from httprunner import exceptions
-from httprunner.schema import TestCase, ProjectMeta
+from httprunner.schema import TestCase, ProjectMeta, TestSuite
 
 try:
     # PyYAML version >= 5.1
@@ -95,6 +95,19 @@ def load_testcase_file(testcase_file: Text) -> TestCase:
     return testcase_obj
 
 
+def load_testsuite(testsuite: Dict) -> TestSuite:
+    path = testsuite["config"]["path"]
+    try:
+        # validate with pydantic TestCase model
+        testsuite_obj = TestSuite.parse_obj(testsuite)
+    except ValidationError as ex:
+        err_msg = f"TestSuite ValidationError:\nfile: {path}\nerror: {ex}"
+        logger.error(err_msg)
+        raise exceptions.TestSuiteFormatError(err_msg)
+
+    return testsuite_obj
+
+
 def load_dot_env_file(dot_env_path: Text) -> Dict:
     """ load .env file.
 
@@ -360,6 +373,14 @@ def init_project_working_directory(test_path: Text) -> Tuple[Text, Text]:
     return debugtalk_path, project_working_directory
 
 
+def get_project_working_directory(test_path: Text) -> Text:
+    global project_working_directory
+    if not project_working_directory:
+        init_project_working_directory(test_path)
+
+    return project_working_directory
+
+
 def load_debugtalk_functions() -> Dict[Text, Callable]:
     """ load project debugtalk.py module functions
         debugtalk.py should be located in project working directory.

httprunner/schema.py

@@ -160,6 +160,18 @@ class PlatformInfo(BaseModel):
     platform: Text
 
 
+class TestCaseRef(BaseModel):
+    name: Text
+    base_url: Text = ""
+    testcase: Text
+    variables: VariablesMapping = {}
+
+
+class TestSuite(BaseModel):
+    config: TConfig
+    testcases: List[TestCaseRef]
+
+
 class Stat(BaseModel):
     total: int = 0
     success: int = 0