Merge pull request #828 from httprunner/leo_dev

2.5.0

**Added**

- feat: add json schema validation for api
- feat: add json schema validation for testcase v1 & v2
- feat: add json schema validation for testsuite v1 & v2

**Changed**

- refactor: use loader.load_cases to validate test files
- refactor: use is_test_path to check if path is valid json/yaml file or a existed directory
- refactor: use is_test_content to check if data_structure is apis/testcases/testsuites

docs/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Release History
 
+## 2.5.0 (2020-01-01)
+
+**Added**
+
+- feat: add json schema validation for api
+- feat: add json schema validation for testcase v1 & v2
+- feat: add json schema validation for testsuite v1 & v2
+
+**Changed**
+
+- refactor: use loader.load_cases to validate test files
+- refactor: use is_test_path to check if path is valid json/yaml file or a existed directory
+- refactor: use is_test_content to check if data_structure is apis/testcases/testsuites
+
 ## 2.4.9 (2019-12-29)
 
 **Added**

httprunner/api.py

@@ -289,9 +289,9 @@ class HttpRunner(object):
 
         """
         logger.log_info("HttpRunner version: {}".format(__version__))
-        if loader.is_testcase_path(path_or_tests):
+        if loader.is_test_path(path_or_tests):
             return self.run_path(path_or_tests, dot_env_path, mapping)
-        elif loader.is_testcases(path_or_tests):
+        elif loader.is_test_content(path_or_tests):
             return self.run_tests(path_or_tests)
         else:
             raise exceptions.ParamsError("Invalid testcase path or testcases: {}".format(path_or_tests))

httprunner/cli.py

@@ -4,11 +4,11 @@ import sys
 
 from sentry_sdk import capture_exception
 
-from httprunner import __description__, __version__
+from httprunner import __description__, __version__, exceptions
 from httprunner.api import HttpRunner
 from httprunner.compat import is_py2
-from httprunner.loader import validate_json_file
+from httprunner.loader import load_cases
-from httprunner.logger import color_print
+from httprunner.logger import color_print, log_error
 from httprunner.report import gen_html_report
 from httprunner.utils import (create_scaffold, get_python2_retire_msg,
                               prettify_json_file)
@@ -73,8 +73,17 @@ def main():
         sys.exit(0)
 
     if args.validate:
-        validate_json_file(args.validate)
+        for validate_path in args.validate:
+            try:
+                color_print("validate test file: {}".format(validate_path), "GREEN")
+                load_cases(validate_path, args.dot_env_path)
+            except exceptions.MyBaseError as ex:
+                log_error(str(ex))
+                continue
+
+        color_print("done!", "BLUE")
         sys.exit(0)
+
     if args.prettify:
         prettify_json_file(args.prettify)
         sys.exit(0)

httprunner/loader/__init__.py

@@ -1,22 +1,22 @@
 """
 HttpRunner loader
 
-- check: validate testcase data structure with JSON schema (TODO)
+- check: validate api/testcase/testsuite data structure with JSON schema
 - locate: locate debugtalk.py, make it's dir as project root path
 - load: load testcase files and relevant data, including debugtalk.py, .env, yaml/json api/testcases, csv, etc.
 - buildup: assemble loaded content to httprunner testcase/testsuite data structure
 
 """
 
-from httprunner.loader.check import is_testcase_path, is_testcases, validate_json_file
+from httprunner.loader.check import is_test_path, is_test_content, JsonSchemaChecker
 from httprunner.loader.locate import get_project_working_directory as get_pwd
 from httprunner.loader.load import load_csv_file, load_builtin_functions
 from httprunner.loader.buildup import load_cases, load_project_data
 
 __all__ = [
-    "is_testcase_path",
+    "is_test_path",
-    "is_testcases",
+    "is_test_content",
-    "validate_json_file",
+    "JsonSchemaChecker",
     "get_pwd",
     "load_csv_file",
     "load_builtin_functions",

httprunner/loader/buildup.py

@@ -2,6 +2,7 @@ import importlib
 import os
 
 from httprunner import exceptions, logger, utils
+from httprunner.loader.check import JsonSchemaChecker
 from httprunner.loader.load import load_module_functions, load_file, load_dot_env_file, \
     load_folder_files
 from httprunner.loader.locate import init_project_working_directory, get_project_working_directory
@@ -174,6 +175,7 @@ def load_testcase(raw_testcase):
             }
 
     """
+    JsonSchemaChecker.validate_testcase_v1_format(raw_testcase)
     config = {}
     tests = []
 
@@ -224,6 +226,7 @@ def load_testcase_v2(raw_testcase):
             }
 
     """
+    JsonSchemaChecker.validate_testcase_v2_format(raw_testcase)
     raw_teststeps = raw_testcase.pop("teststeps")
     raw_testcase["teststeps"] = [
         load_teststep(teststep)
@@ -279,11 +282,12 @@ def load_testsuite(raw_testsuite):
             }
 
     """
-    raw_testcases = raw_testsuite.pop("testcases")
+    raw_testcases = raw_testsuite["testcases"]
-    raw_testsuite["testcases"] = {}
 
     if isinstance(raw_testcases, dict):
-        # make compatible with version < 2.2.0
+        # format version 1, make compatible with version < 2.2.0
+        JsonSchemaChecker.validate_testsuite_v1_format(raw_testsuite)
+        raw_testsuite["testcases"] = {}
         for name, raw_testcase in raw_testcases.items():
             __extend_with_testcase_ref(raw_testcase)
             raw_testcase.setdefault("name", name)
@@ -291,6 +295,8 @@ def load_testsuite(raw_testsuite):
 
     elif isinstance(raw_testcases, list):
         # format version 2, implemented in 2.2.0
+        JsonSchemaChecker.validate_testsuite_v2_format(raw_testsuite)
+        raw_testsuite["testcases"] = {}
         for raw_testcase in raw_testcases:
             __extend_with_testcase_ref(raw_testcase)
             testcase_name = raw_testcase["name"]
@@ -343,7 +349,6 @@ def load_test_file(path):
 
         if "testcases" in raw_content:
             # file_type: testsuite
-            # TODO: add json schema validation for testsuite
             loaded_content = load_testsuite(raw_content)
             loaded_content["path"] = path
             loaded_content["type"] = "testsuite"
@@ -356,7 +361,7 @@ def load_test_file(path):
 
         elif "request" in raw_content:
             # file_type: api
-            # TODO: add json schema validation for api
+            JsonSchemaChecker.validate_api_format(raw_content)
             loaded_content = raw_content
             loaded_content["path"] = path
             loaded_content["type"] = "api"
@@ -368,7 +373,6 @@ def load_test_file(path):
     elif isinstance(raw_content, list) and len(raw_content) > 0:
         # file_type: testcase
         # make compatible with version < 2.2.0
-        # TODO: add json schema validation for testcase
         loaded_content = load_testcase(raw_content)
         loaded_content["path"] = path
         loaded_content["type"] = "testcase"

httprunner/loader/check.py

@@ -1,193 +1,206 @@
-import io
 import json
 import os
-import types
 
-from httprunner import logger, exceptions
+import jsonschema
+
+from httprunner import exceptions, logger
+
+schemas_root_dir = os.path.join(os.path.dirname(__file__), "schemas")
+common_schema_path = os.path.join(schemas_root_dir, "common.schema.json")
+api_schema_path = os.path.join(schemas_root_dir, "api.schema.json")
+testcase_schema_v1_path = os.path.join(schemas_root_dir, "testcase.schema.v1.json")
+testcase_schema_v2_path = os.path.join(schemas_root_dir, "testcase.schema.v2.json")
+testsuite_schema_v1_path = os.path.join(schemas_root_dir, "testsuite.schema.v1.json")
+testsuite_schema_v2_path = os.path.join(schemas_root_dir, "testsuite.schema.v2.json")
+
+with open(api_schema_path) as f:
+    api_schema = json.load(f)
+
+with open(common_schema_path) as f:
+    common_schema = json.load(f)
+    resolver = jsonschema.RefResolver("file://{}/".format(os.path.abspath(schemas_root_dir)), common_schema)
+
+with open(testcase_schema_v1_path) as f:
+    testcase_schema_v1 = json.load(f)
+
+with open(testcase_schema_v2_path) as f:
+    testcase_schema_v2 = json.load(f)
+
+with open(testsuite_schema_v1_path) as f:
+    testsuite_schema_v1 = json.load(f)
+
+with open(testsuite_schema_v2_path) as f:
+    testsuite_schema_v2 = json.load(f)
 
 
-# TODO: validate data format with JSON schema
+class JsonSchemaChecker(object):
 
-def is_testcase(data_structure):
+    @staticmethod
-    """ check if data_structure is a testcase.
+    def validate_format(content, scheme):
+        """ check api/testcase/testsuite format if valid
+        """
+        try:
+            jsonschema.validate(content, scheme, resolver=resolver)
+        except jsonschema.exceptions.ValidationError as ex:
+            logger.log_error(str(ex))
+            raise exceptions.FileFormatError
+
+        return True
+
+    @staticmethod
+    def validate_api_format(content):
+        """ check api format if valid
+        """
+        return JsonSchemaChecker.validate_format(content, api_schema)
+
+    @staticmethod
+    def validate_testcase_v1_format(content):
+        """ check testcase format v1 if valid
+        """
+        return JsonSchemaChecker.validate_format(content, testcase_schema_v1)
+
+    @staticmethod
+    def validate_testcase_v2_format(content):
+        """ check testcase format v2 if valid
+        """
+        return JsonSchemaChecker.validate_format(content, testcase_schema_v2)
+
+    @staticmethod
+    def validate_testsuite_v1_format(content):
+        """ check testsuite format v1 if valid
+        """
+        return JsonSchemaChecker.validate_format(content, testsuite_schema_v1)
+
+    @staticmethod
+    def validate_testsuite_v2_format(content):
+        """ check testsuite format v2 if valid
+        """
+        return JsonSchemaChecker.validate_format(content, testsuite_schema_v2)
+
+
+def is_test_path(path):
+    """ check if path is valid json/yaml file path or a existed directory.
 
     Args:
-        data_structure (dict): testcase should always be in the following data structure:
+        path (str/list/tuple): file path/directory or file path list.
-
-            {
-                "config": {
-                    "name": "desc1",
-                    "variables": [],    # optional
-                    "request": {}       # optional
-                },
-                "teststeps": [
-                    test_dict1,
-                    {   # test_dict2
-                        'name': 'test step desc2',
-                        'variables': [],    # optional
-                        'extract': [],      # optional
-                        'validate': [],
-                        'request': {},
-                        'function_meta': {}
-                    }
-                ]
-            }
-
-    Returns:
-        bool: True if data_structure is valid testcase, otherwise False.
-
-    """
-    # TODO: replace with JSON schema validation
-    if not isinstance(data_structure, dict):
-        return False
-
-    if "teststeps" not in data_structure:
-        return False
-
-    if not isinstance(data_structure["teststeps"], list):
-        return False
-
-    return True
-
-
-def is_testcases(data_structure):
-    """ check if data_structure is testcase or testcases list.
-
-    Args:
-        data_structure (dict): testcase(s) should always be in the following data structure:
-            {
-                "project_mapping": {
-                    "PWD": "XXXXX",
-                    "functions": {},
-                    "env": {}
-                },
-                "testcases": [
-                    {   # testcase data structure
-                        "config": {
-                            "name": "desc1",
-                            "path": "testcase1_path",
-                            "variables": [],                    # optional
-                        },
-                        "teststeps": [
-                            # test data structure
-                            {
-                                'name': 'test step desc1',
-                                'variables': [],    # optional
-                                'extract': [],      # optional
-                                'validate': [],
-                                'request': {}
-                            },
-                            test_dict_2   # another test dict
-                        ]
-                    },
-                    testcase_dict_2     # another testcase dict
-                ]
-            }
-
-    Returns:
-        bool: True if data_structure is valid testcase(s), otherwise False.
-
-    """
-    if not isinstance(data_structure, dict):
-        return False
-
-    if "testcases" not in data_structure:
-        return False
-
-    testcases = data_structure["testcases"]
-    if not isinstance(testcases, list):
-        return False
-
-    for item in testcases:
-        if not is_testcase(item):
-            return False
-
-    return True
-
-
-def is_testcase_path(path):
-    """ check if path is testcase path or path list.
-
-    Args:
-        path (str/list): file path or file path list.
 
     Returns:
         bool: True if path is valid file path or path list, otherwise False.
 
     """
-    if not isinstance(path, (str, list)):
+    if not isinstance(path, (str, list, tuple)):
         return False
 
-    if isinstance(path, list):
+    elif isinstance(path, (list, tuple)):
         for p in path:
-            if not is_testcase_path(p):
+            if not is_test_path(p):
                 return False
 
-    if isinstance(path, str):
+        return True
+
+    else:
+        # path is string
         if not os.path.exists(path):
             return False
 
-        # TODO: check file format if valid
+        # path exists
-
+        if os.path.isfile(path):
-    return True
+            # path is a file
+            file_suffix = os.path.splitext(path)[1].lower()
+            if file_suffix not in ['.json', '.yaml', '.yml']:
+                # path is not json/yaml file
+                return False
+            else:
+                return True
+        elif os.path.isdir(path):
+            # path is a directory
+            return True
+        else:
+            # path is neither a folder nor a file, maybe a symbol link or something else
+            return False
 
 
-def check_testcase_format(file_path, content):
+def is_test_content(data_structure):
-    """ check testcase format if valid
+    """ check if data_structure is apis/testcases/testsuites.
+
+    Args:
+        data_structure (dict): should include keys, apis or testcases or testsuites
+
+    Returns:
+        bool: True if data_structure is valid apis/testcases/testsuites, otherwise False.
+
     """
-    # TODO: replace with JSON schema validation
+    if not isinstance(data_structure, dict):
-    if not content:
+        return False
-        # testcase file content is empty
-        err_msg = u"Testcase file content is empty: {}".format(file_path)
-        logger.log_error(err_msg)
-        raise exceptions.FileFormatError(err_msg)
 
-    elif not isinstance(content, (list, dict)):
+    if "apis" in data_structure:
-        # testcase file content does not match testcase format
+        # maybe a group of api content
-        err_msg = u"Testcase file content format invalid: {}".format(file_path)
+        apis = data_structure["apis"]
-        logger.log_error(err_msg)
+        if not isinstance(apis, list):
-        raise exceptions.FileFormatError(err_msg)
+            return False
 
-
+        for item in apis:
-def validate_json_file(file_list):
+            is_testcase = False
-    """ validate JSON testcase format
-    """
-    for json_file in set(file_list):
-        if not json_file.endswith(".json"):
-            logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file))
-            continue
-
-        logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN")
-
-        with io.open(json_file) as stream:
             try:
-                json.load(stream)
+                JsonSchemaChecker.validate_api_format(item)
-            except ValueError as e:
+                is_testcase = True
-                raise SystemExit(e)
+            except exceptions.FileFormatError:
+                pass
 
-        print("OK")
+            if not is_testcase:
+                return False
 
+        return True
 
-def is_function(item):
+    elif "testcases" in data_structure:
-    """ Takes item object, returns True if it is a function.
+        # maybe a testsuite, containing a group of testcases
-    """
+        testcases = data_structure["testcases"]
-    return isinstance(item, types.FunctionType)
+        if not isinstance(testcases, list):
+            return False
 
+        for item in testcases:
+            is_testcase = False
+            try:
+                JsonSchemaChecker.validate_testcase_v2_format(item)
+                is_testcase = True
+            except exceptions.FileFormatError:
+                pass
 
-def is_variable(tup):
+            try:
-    """ Takes (name, object) tuple, returns True if it is a variable.
+                JsonSchemaChecker.validate_testcase_v2_format(item)
-    """
+                is_testcase = True
-    name, item = tup
+            except exceptions.FileFormatError:
-    if callable(item):
+                pass
-        # function or class
+
+            if not is_testcase:
+                return False
+
+        return True
+
+    elif "testsuites" in data_structure:
+        # maybe a group of testsuites
+        testsuites = data_structure["testsuites"]
+        if not isinstance(testsuites, list):
+            return False
+
+        for item in testsuites:
+            is_testcase = False
+            try:
+                JsonSchemaChecker.validate_testsuite_v1_format(item)
+                is_testcase = True
+            except exceptions.FileFormatError:
+                pass
+
+            try:
+                JsonSchemaChecker.validate_testsuite_v2_format(item)
+                is_testcase = True
+            except exceptions.FileFormatError:
+                pass
+
+            if not is_testcase:
+                return False
+
+        return True
+
+    else:
         return False
-
-    if isinstance(item, types.ModuleType):
-        # imported module
-        return False
-
-    if name.startswith("_"):
-        # private property
-        return False
-
-    return True

httprunner/loader/load.py

@@ -2,12 +2,12 @@ import csv
 import io
 import json
 import os
+import types
 
 import yaml
 
 from httprunner import builtin
 from httprunner import exceptions, logger, utils
-from httprunner.loader.check import check_testcase_format, is_function
 from httprunner.loader.locate import get_project_working_directory
 
 try:
@@ -22,8 +22,12 @@ def _load_yaml_file(yaml_file):
     """ load yaml file and check file content format
     """
     with io.open(yaml_file, 'r', encoding='utf-8') as stream:
-        yaml_content = yaml.load(stream)
+        try:
-        check_testcase_format(yaml_file, yaml_content)
+            yaml_content = yaml.load(stream)
+        except yaml.YAMLError as ex:
+            logger.log_error(str(ex))
+            raise exceptions.FileFormatError
+
         return yaml_content
 
 
@@ -38,7 +42,6 @@ def _load_json_file(json_file):
             logger.log_error(err_msg)
             raise exceptions.FileFormatError(err_msg)
 
-        check_testcase_format(json_file, json_content)
         return json_content
 
 
@@ -203,7 +206,7 @@ def load_module_functions(module):
     module_functions = {}
 
     for name, item in vars(module).items():
-        if is_function(item):
+        if isinstance(item, types.FunctionType):
             module_functions[name] = item
 
     return module_functions

httprunner/loader/schemas/api.schema.json

@@ -0,0 +1,35 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "description": "httprunner api schema definition",
+    "type": "object",
+    "properties": {
+        "name": {
+            "$ref": "common.schema.json#/definitions/name"
+        },
+        "base_url": {
+            "$ref": "common.schema.json#/definitions/base_url"
+        },
+        "variables": {
+            "$ref": "common.schema.json#/definitions/variables"
+        },
+        "request": {
+            "$ref": "common.schema.json#/definitions/request"
+        },
+        "setup_hooks": {
+            "$ref": "common.schema.json#/definitions/hook"
+        },
+        "teardown_hooks": {
+            "$ref": "common.schema.json#/definitions/hook"
+        },
+        "extract": {
+            "$ref": "common.schema.json#/definitions/extract"
+        },
+        "validate": {
+            "$ref": "common.schema.json#/definitions/validate"
+        }
+    },
+    "required": [
+        "name",
+        "request"
+    ]
+}

httprunner/loader/schemas/common.schema.json

@@ -0,0 +1,252 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "description": "common json schema definitions for httprunner api/testcase/testsuite",
+    "definitions": {
+        "name": {
+            "description": "used as api/teststep/testcase/testsuite identification",
+            "type": "string"
+        },
+        "base_url": {
+            "description": "The base_url will be used with relative URI",
+            "type": "string"
+        },
+        "variables": {
+            "description": "define variables for api/teststep/testcase/testsuite",
+            "oneOf": [
+                {
+                    "type": "object"
+                },
+                {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "maxProperties": 1,
+                        "minProperties": 1
+                    }
+                }
+            ]
+        },
+        "config": {
+            "description": "used in testcase/testsuite to configure common fields",
+            "type": "object",
+            "properties": {
+                "name": {
+                    "$ref": "#/definitions/name"
+                },
+                "base_url": {
+                    "$ref": "#/definitions/base_url"
+                },
+                "variables": {
+                    "$ref": "#/definitions/variables"
+                },
+                "setup_hooks": {
+                    "$ref": "#/definitions/hook"
+                },
+                "teardown_hooks": {
+                    "$ref": "#/definitions/hook"
+                },
+                "verify": {
+                    "description": "configure verify for current testcase/testsuite",
+                    "oneOf": [
+                        {
+                            "description": "whether we verify the server’s TLS certificate",
+                            "type": "boolean"
+                        },
+                        {
+                            "description": "path to a CA bundle to use",
+                            "type": "string"
+                        }
+                    ]
+                }
+            },
+            "required": ["name"]
+        },
+        "hook": {
+            "description": "used to define setup_hooks/teardown_hooks for api/teststep/testcase",
+            "type": "array",
+            "items": {
+                "type": "string"
+            }
+        },
+        "request": {
+            "description": "used to define a api request. properties is the same as python package `requests.request`",
+            "type": "object",
+            "properties": {
+                "method": {
+                    "type": "string",
+                    "description": "request method",
+                    "enum": [
+                        "GET",
+                        "POST",
+                        "OPTIONS",
+                        "HEAD",
+                        "PUT",
+                        "PATCH",
+                        "DELETE"
+                    ]
+                },
+                "url": {
+                    "description": "request url, may be absolute or relative URI",
+                    "type": "string"
+                },
+                "params": {
+                    "description": "query string for request url",
+                    "type": "object"
+                },
+                "data": {
+                    "anyOf": [
+                        {
+                            "description": "request body in json format",
+                            "type": "object"
+                        },
+                        {
+                            "description": "request body in application/x-www-form-urlencoded format, e.g. a=1&b=2",
+                            "type": "string"
+                        },
+                        {
+                            "description": "request body in string format, e.g. '${prepare_data($a, $b)}'",
+                            "type": "string"
+                        }
+                    ]
+                },
+                "json": {
+                    "description": "request body in json format",
+                    "type": "object"
+                },
+                "headers": {
+                    "description": "request headers",
+                    "type": "object"
+                },
+                "cookies": {
+                    "description": "request cookies",
+                    "type": "object"
+                },
+                "files": {
+                    "description": "request files, used to upload files",
+                    "type": "object"
+                },
+                "auth": {
+                    "description": "Auth tuple to enable Basic/Digest/Custom HTTP Auth.",
+                    "type": "array"
+                },
+                "timeout": {
+                    "description": "How many seconds to wait for the server to send data before giving up",
+                    "type": "number"
+                },
+                "allow_redirects": {
+                    "description": "Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to True",
+                    "type": "boolean"
+                },
+                "proxies": {
+                    "description": "Dictionary mapping protocol to the URL of the proxy",
+                    "type": "object"
+                },
+                "verify": {
+                    "description": "configure verify for current api/teststep",
+                    "oneOf": [
+                        {
+                            "description": "whether we verify the server’s TLS certificate",
+                            "type": "boolean"
+                        },
+                        {
+                            "description": "path to a CA bundle to use",
+                            "type": "string"
+                        }
+                    ]
+                },
+                "stream": {
+                    "description": "if False, the response content will be immediately downloaded.",
+                    "type": "boolean"
+                },
+                "cert": {
+                    "oneOf": [
+                        {
+                            "description": "path to ssl client cert file (.pem)",
+                            "type": "string"
+                        },
+                        {
+                            "description": "('cert', 'key') pair",
+                            "type": "array",
+                            "maxItems": 2,
+                            "minItems": 2,
+                            "items": {
+                                "type": "string"
+                            }
+                        }
+                    ]
+                },
+                "upload": {
+                    "description": "upload files",
+                    "type": "object"
+                }
+            },
+            "required": [
+                "method",
+                "url"
+            ]
+        },
+        "extract": {
+            "description": "used to extract session variables for later requests",
+            "oneOf": [
+                {
+                    "type": "object",
+                    "patternProperties": {
+                        "^[A-Za-z_][A-Za-z0-9_]*$": {
+                            "description": "extraction rule for session variable, maybe in jsonpath/regex/jmespath",
+                            "type": "string"
+                        }
+                    }
+                },
+                {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "patternProperties": {
+                            "^[A-Za-z_][A-Za-z0-9_]*$": {
+                                "description": "extraction rule for session variable, maybe in jsonpath/regex/jmespath",
+                                "type": "string"
+                            }
+                        },
+                        "minProperties": 1,
+                        "maxProperties": 1
+                    }
+                }
+            ]
+        },
+        "validate": {
+            "description": "used to validate response fields",
+            "type": "array",
+            "items": {
+                "description": "one validator definition",
+                "oneOf": [
+                    {
+                        "type": "object",
+                        "properties": {
+                            "check": {
+                                "type": "string"
+                            },
+                            "comparator": {
+                                "type": "string"
+                            },
+                            "expect": {
+                                "description": "expected value"
+                            }
+                        },
+                        "required": ["check", "expect"]
+                    },
+                    {
+                        "type": "object",
+                        "patternProperties": {
+                            "^[A-Za-z_][A-Za-z0-9_]*$": {
+                                "description": "validate_func_name: [check_value, expect_value]",
+                                "type": "array",
+                                "minItems": 2,
+                                "maxItems": 2
+                            }
+                        }
+                    }
+                ]
+            }
+        }
+    }
+}

httprunner/loader/schemas/testcase.schema.v1.json

@@ -0,0 +1,138 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "description": "httprunner testcase schema v1 definition",
+    "type": "array",
+    "definitions": {
+        "test": {
+            "type": "object",
+            "oneOf": [
+                {
+                    "properties": {
+                        "name": {
+                            "$ref": "common.schema.json#/definitions/name"
+                        },
+                        "request": {
+                            "description": "define api request directly",
+                            "$ref": "common.schema.json#/definitions/request"
+                        },
+                        "variables": {
+                            "$ref": "common.schema.json#/definitions/variables"
+                        },
+                        "extract": {
+                            "$ref": "common.schema.json#/definitions/extract"
+                        },
+                        "validate": {
+                            "$ref": "common.schema.json#/definitions/validate"
+                        },
+                        "setup_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        },
+                        "teardown_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        }
+                    },
+                    "required": [
+                        "name",
+                        "request"
+                    ]
+                },
+                {
+                    "properties": {
+                        "name": {
+                            "$ref": "common.schema.json#/definitions/name"
+                        },
+                        "api": {
+                            "description": "api reference, value is api file relative path",
+                            "type": "string"
+                        },
+                        "variables": {
+                            "$ref": "common.schema.json#/definitions/variables"
+                        },
+                        "extract": {
+                            "oneOf": [
+                                {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "string"
+                                    }
+                                },
+                                {
+                                    "$ref": "common.schema.json#/definitions/extract"
+                                }
+                            ]
+                        },
+                        "validate": {
+                            "$ref": "common.schema.json#/definitions/validate"
+                        },
+                        "setup_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        },
+                        "teardown_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        }
+                    },
+                    "required": [
+                        "name",
+                        "api"
+                    ]
+                },
+                {
+                    "properties": {
+                        "name": {
+                            "$ref": "common.schema.json#/definitions/name"
+                        },
+                        "testcase": {
+                            "description": "testcase reference, value is testcase file relative path",
+                            "type": "string"
+                        },
+                        "variables": {
+                            "$ref": "common.schema.json#/definitions/variables"
+                        },
+                        "extract": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            }
+                        },
+                        "setup_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        },
+                        "teardown_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        }
+                    },
+                    "required": [
+                        "name",
+                        "testcase"
+                    ]
+                }
+            ]
+        }
+    },
+    "items": {
+        "type": "object",
+        "oneOf": [
+            {
+                "type": "object",
+                "properties": {
+                    "config": {
+                        "$ref": "common.schema.json#/definitions/config"
+                    }
+                },
+                "additionalProperties": false
+            },
+            {
+                "type": "object",
+                "properties": {
+                    "test": {
+                        "$ref": "testcase.schema.v1.json#/definitions/test"
+                    }
+                },
+                "additionalProperties": false
+            }
+        ],
+        "minProperties": 1,
+        "maxProperties": 1
+    },
+    "minItems": 2
+}

httprunner/loader/schemas/testcase.schema.v2.json

@@ -0,0 +1,129 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "description": "httprunner testcase schema v2 definition",
+    "type": "object",
+    "definitions": {
+        "teststep": {
+            "type": "object",
+            "oneOf": [
+                {
+                    "properties": {
+                        "name": {
+                            "$ref": "common.schema.json#/definitions/name"
+                        },
+                        "request": {
+                            "description": "define api request directly",
+                            "$ref": "common.schema.json#/definitions/request"
+                        },
+                        "variables": {
+                            "$ref": "common.schema.json#/definitions/variables"
+                        },
+                        "extract": {
+                            "$ref": "common.schema.json#/definitions/extract"
+                        },
+                        "validate": {
+                            "$ref": "common.schema.json#/definitions/validate"
+                        },
+                        "setup_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        },
+                        "teardown_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        }
+                    },
+                    "required": [
+                        "name",
+                        "request"
+                    ]
+                },
+                {
+                    "properties": {
+                        "name": {
+                            "$ref": "common.schema.json#/definitions/name"
+                        },
+                        "api": {
+                            "description": "api reference, value is api file relative path",
+                            "type": "string"
+                        },
+                        "variables": {
+                            "$ref": "common.schema.json#/definitions/variables"
+                        },
+                        "extract": {
+                            "oneOf": [
+                                {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "string"
+                                    }
+                                },
+                                {
+                                    "$ref": "common.schema.json#/definitions/extract"
+                                }
+                            ]
+                        },
+                        "validate": {
+                            "$ref": "common.schema.json#/definitions/validate"
+                        },
+                        "setup_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        },
+                        "teardown_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        }
+                    },
+                    "required": [
+                        "name",
+                        "api"
+                    ]
+                },
+                {
+                    "properties": {
+                        "name": {
+                            "$ref": "common.schema.json#/definitions/name"
+                        },
+                        "testcase": {
+                            "description": "testcase reference, value is testcase file relative path",
+                            "type": "string"
+                        },
+                        "variables": {
+                            "$ref": "common.schema.json#/definitions/variables"
+                        },
+                        "extract": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            }
+                        },
+                        "setup_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        },
+                        "teardown_hooks": {
+                            "$ref": "common.schema.json#/definitions/hook"
+                        }
+                    },
+                    "required": [
+                        "name",
+                        "testcase"
+                    ]
+                }
+            ]
+        }
+    },
+    "properties": {
+        "config": {
+            "$ref": "common.schema.json#/definitions/config"
+        },
+        "teststeps": {
+            "description": "teststep of a testcase",
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "$ref": "testcase.schema.v2.json#/definitions/teststep"
+            }
+        }
+    },
+    "required": [
+        "config",
+        "teststeps"
+    ]
+}

httprunner/loader/schemas/testsuite.schema.v1.json

@@ -0,0 +1,66 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "description": "httprunner testsuite schema v1 definition",
+    "type": "object",
+    "definitions": {
+        "testcase": {
+            "type": "object",
+            "properties": {
+                "name": {
+                    "$ref": "common.schema.json#/definitions/name"
+                },
+                "variables": {
+                    "$ref": "common.schema.json#/definitions/variables"
+                },
+                "parameters": {
+                    "description": "generate cartesian product variables with parameters, each group of variables will be run once",
+                    "type": "object"
+                },
+                "testcase": {
+                    "description": "testcase reference, value is testcase file relative path",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "testcase"
+            ]
+        }
+    },
+    "properties": {
+        "config": {
+            "$ref": "common.schema.json#/definitions/config"
+        },
+        "testcases": {
+            "description": "testcase of a testsuite",
+            "type": "object",
+            "minProperties": 1,
+            "patternProperties": {
+                ".*": {
+                    "description": "testcase definition",
+                    "$ref": "testsuite.schema.v1.json#/definitions/testcase"
+                }
+            }
+        }
+    },
+    "required": [
+        "config",
+        "testcases"
+    ],
+    "examples": [
+        {
+            "config": {
+                "name": "testsuite name"
+            },
+            "testcases": {
+                "testcase 1": {
+                    "name": "xxx",
+                    "testcase": "/path/to/testcase1"
+                },
+                "testcase 2": {
+                    "name": "xxx",
+                    "testcase": "/path/to/testcase2"
+                }
+            }
+        }
+    ]
+}

httprunner/loader/schemas/testsuite.schema.v2.json

@@ -0,0 +1,46 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "description": "httprunner testsuite schema v2 definition",
+    "type": "object",
+    "definitions": {
+        "testcase": {
+            "type": "object",
+            "properties": {
+                "name": {
+                     "$ref": "common.schema.json#/definitions/name"
+                },
+                "variables": {
+                    "$ref": "common.schema.json#/definitions/variables"
+                },
+                "parameters": {
+                    "description": "generate cartesian product variables with parameters, each group of variables will be run once",
+                    "type": "object"
+                },
+                "testcase": {
+                    "description": "testcase reference, value is testcase file relative path",
+                    "type": "string"
+                }
+            },
+            "required": [
+                "testcase"
+            ]
+        }
+    },
+    "properties": {
+        "config": {
+            "$ref": "common.schema.json#/definitions/config"
+        },
+        "testcases": {
+            "description": "testcase of a testsuite",
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "$ref": "testsuite.schema.v2.json#/definitions/testcase"
+            }
+        }
+    },
+    "required": [
+        "config",
+        "testcases"
+    ]
+}

httprunner/loader/schemas/v2/api.schema.json

@@ -1,42 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/api.schema.json",
-    "title": "Api for httprunner",
-    "description": "Api for httprunner",
-    "type": "object",
-    "properties": {
-        "name": {
-            "type": "string",
-            "description": "Name of the api"
-        },
-        "base_url": {
-            "type": "string",
-            "description": "The base_url will be added before a relative URI"
-        },
-        "request": {
-            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/request.schema.json"
-        },
-        "variables": {
-            "type": "object",
-            "description": "Variables for the api"
-        },
-        "extract": {
-            "type": "array",
-            "description": "Extract rules",
-            "items": {
-                "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/extract.schema.json"
-            }
-        },
-        "validate": {
-            "type": "array",
-            "description": "Validate rules",
-            "items": {
-                "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/validate.schema.json"
-            }
-        }
-    },
-    "required": [
-        "name",
-        "request"
-    ]
-}

httprunner/loader/schemas/v2/common/config.schema.json

@@ -1,25 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/config.schema.json",
-    "title": "Config for httprunner",
-    "description": "Used in teststep/testcase/testsuite",
-    "type": "object",
-    "properties": {
-        "name": {
-            "type": "string"
-        },
-        "base_url": {
-            "type": "string",
-            "description": "The base_url will be added before a relative URI"
-        },
-        "variables": {
-            "type": "object"
-        },
-        "setup_hooks": {
-            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json"
-        },
-        "teardown_hooks": {
-            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json"
-        }
-    }
-}

httprunner/loader/schemas/v2/common/extract.schema.json

@@ -1,13 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/extract.schema.json",
-    "title": "Extract rules for httprunner",
-    "description": "Used to extract variables for later requests",
-    "type": "object",
-    "patternProperties": {
-        ".*": {
-            "description": "extracted_variable_name: extract_rule",
-            "type": "string"
-        }
-    }
-}

httprunner/loader/schemas/v2/common/hook.schema.json

@@ -1,10 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json",
-    "title": "setup_hooks or teardown_hooks for httprunner",
-    "description": "Define setup_hooks or teardown_hooks for httprunner",
-    "type": "array",
-    "items": {
-        "type": "string"
-    }
-}

httprunner/loader/schemas/v2/common/request.schema.json

@@ -1,93 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/request.schema.json",
-    "title": "request for httprunner",
-    "description": "Used to define a api or used in a teststep. Same parameters as python's package 'requests'",
-    "type": "object",
-    "properties": {
-        "method": {
-            "type": "string",
-            "description": "Request method",
-            "enum": [
-                "GET",
-                "POST",
-                "OPTIONS",
-                "HEAD",
-                "PUT",
-                "PATCH",
-                "DELETE"
-            ]
-        },
-        "url": {
-            "description": "Request url",
-            "type": "string"
-        },
-        "params": {
-            "type": "object",
-            "description": "Used to define the parameters of a 'GET' request"
-        },
-        "data": {
-            "type": "object",
-            "description": "Used to define the parameters of a 'POST' request in the application/x-www-form-urlencoded format"
-        },
-        "json": {
-            "type": "object",
-            "description": "Used to define the parameters of a 'POST' request in the application/json format"
-        },
-        "headers": {
-            "description": "Request headers",
-            "type": "object"
-        },
-        "cookies": {
-            "description": "Request cookies",
-            "type": "object"
-        },
-        "files": {
-            "type": "object"
-        },
-        "auth": {
-            "type": "array"
-        },
-        "timeout": {
-            "type": "number"
-        },
-        "allow_redirects": {
-            "type": "boolean"
-        },
-        "proxies": {
-            "type": "object"
-        },
-        "verify": {
-            "oneOf": [
-                {
-                    "type": "boolean"
-                },
-                {
-                    "type": "string"
-                }
-            ]
-        },
-        "stream": {
-            "type": "boolean"
-        },
-        "cert": {
-            "oneOf": [
-                {
-                    "type": "string"
-                },
-                {
-                    "type": "array",
-                    "maxItems": 2,
-                    "minItems": 2,
-                    "items": {
-                        "type": "string"
-                    }
-                }
-            ]
-        }
-    },
-    "required": [
-        "method",
-        "url"
-    ]
-}

httprunner/loader/schemas/v2/common/validate.schema.json

@@ -1,15 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/validate.schema.json",
-    "title": "Validate rule of httprunner",
-    "description": "Validate rule of httprunner",
-    "type": "object",
-    "patternProperties": {
-        ".*": {
-            "type": "array",
-            "description": "validate_function_name: [check_value, expect_value]",
-            "minItems": 2,
-            "maxItems": 2
-        }
-    }
-}

httprunner/loader/schemas/v2/testcase.schema.json

@@ -1,104 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/testcase.schema.json",
-    "title": "Testcase for httprunner",
-    "description": "Testcase for httprunner",
-    "type": "object",
-    "properties": {
-        "config": {
-            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/config.schema.json"
-        },
-        "teststeps": {
-            "description": "Teststep of a testcase",
-            "type": "array",
-            "items": {
-                "$ref": "#/definitions/teststep"
-            }
-        }
-    },
-    "required": [
-        "teststeps"
-    ],
-    "definitions": {
-        "teststep": {
-            "type": "object",
-            "oneOf": [
-                {
-                    "properties": {
-                        "name": {
-                            "description": "Teststep name",
-                            "type": "string"
-                        },
-                        "api": {
-                            "description": "Api reference, it's usually the relative path of the api",
-                            "type": "string"
-                        },
-                        "variables": {
-                            "type": "object"
-                        },
-                        "extract": {
-                            "description": "Extract rules",
-                            "type": "array",
-                            "items": {
-                                "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/extract.schema.json"
-                            }
-                        },
-                        "validate": {
-                            "description": "Validate rules",
-                            "type": "array",
-                            "items": {
-                                "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/validate.schema.json"
-                            }
-                        },
-                        "setup_hooks": {
-                            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json"
-                        },
-                        "teardown_hooks": {
-                            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json"
-                        }
-                    },
-                    "required": [
-                        "api"
-                    ]
-                },
-                {
-                    "properties": {
-                        "name": {
-                            "description": "Teststep name",
-                            "type": "string"
-                        },
-                        "request": {
-                            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/request.schema.json"
-                        },
-                        "variables": {
-                            "type": "object"
-                        },
-                        "extract": {
-                            "description": "Extract rules",
-                            "type": "array",
-                            "items": {
-                                "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/extract.schema.json"
-                            }
-                        },
-                        "validate": {
-                            "description": "Validate rules",
-                            "type": "array",
-                            "items": {
-                                "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/validate.schema.json"
-                            }
-                        },
-                        "setup_hooks": {
-                            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json"
-                        },
-                        "teardown_hooks": {
-                            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json"
-                        }
-                    },
-                    "required": [
-                        "request"
-                    ]
-                }
-            ]
-        }
-    }
-}

httprunner/loader/schemas/v2/testsuite.schema.json

@@ -1,58 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/testsuite.schema.json",
-    "title": "Testsuite for httprunner",
-    "description": "Testsuite for httprunner",
-    "type": "object",
-    "properties": {
-        "config": {
-            "allOf": [
-                {
-                    "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/config.schema.json"
-                }
-            ],
-            "properties": {
-                "verify": {
-                    "type": "boolean"
-                }
-            }
-        },
-        "testcases": {
-            "type": "array",
-            "items": {
-                "$ref": "#/definitions/testcase"
-            }
-        },
-        "setup_hooks": {
-            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json"
-        },
-        "teardown_hooks": {
-            "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json"
-        }
-    },
-    "required": [
-        "testcases"
-    ],
-    "definitions": {
-        "testcase": {
-            "type": "object",
-            "properties": {
-                "name": {
-                    "description": "Testcase name",
-                    "type": "string"
-                },
-                "parameters": {
-                    "description": "Parameters will generate cartesian product variables, each set of variables is tested once",
-                    "type": "object"
-                },
-                "testcase": {
-                    "description": "Testcase reference, it's usually the relative path of the testcase",
-                    "type": "string"
-                }
-            },
-            "required": [
-                "testcase"
-            ]
-        }
-    }
-}

poetry.lock

@@ -1,3 +1,17 @@
+[[package]]
+category = "main"
+description = "Classes Without Boilerplate"
+name = "attrs"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "19.3.0"
+
+[package.extras]
+azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"]
+dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"]
+docs = ["sphinx", "zope.interface"]
+tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
+
 [[package]]
 category = "main"
 description = "Python package for providing Mozilla's CA Bundle."
@@ -41,6 +55,28 @@ version = "4.0.2"
 [package.dependencies]
 colorama = "*"
 
+[[package]]
+category = "main"
+description = "Updated configparser from Python 3.7 for Python 2.6+."
+marker = "python_version < \"3\""
+name = "configparser"
+optional = false
+python-versions = ">=2.6"
+version = "4.0.2"
+
+[package.extras]
+docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
+testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2)", "pytest-flake8", "pytest-black-multipy"]
+
+[[package]]
+category = "main"
+description = "Backports and enhancements for the contextlib module"
+marker = "python_version < \"3\""
+name = "contextlib2"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "0.6.0.post1"
+
 [[package]]
 category = "dev"
 description = "Code coverage measurement for Python"
@@ -80,6 +116,15 @@ Werkzeug = ">=0.7"
 click = ">=2.0"
 itsdangerous = ">=0.21"
 
+[[package]]
+category = "main"
+description = "Backport of the functools module from Python 3.2.3 for use on 2.7 and PyPy."
+marker = "python_version < \"3\""
+name = "functools32"
+optional = false
+python-versions = "*"
+version = "3.2.3-2"
+
 [[package]]
 category = "main"
 description = "Clean single-source support for Python 3 and 2"
@@ -108,6 +153,34 @@ optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 version = "2.8"
 
+[[package]]
+category = "main"
+description = "Read metadata from Python packages"
+marker = "python_version < \"3.8\""
+name = "importlib-metadata"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
+version = "1.3.0"
+
+[package.dependencies]
+zipp = ">=0.5"
+
+[package.dependencies.configparser]
+python = "<3"
+version = ">=3.5"
+
+[package.dependencies.contextlib2]
+python = "<3"
+version = "*"
+
+[package.dependencies.pathlib2]
+python = "<3"
+version = "*"
+
+[package.extras]
+docs = ["sphinx", "rst.linker"]
+testing = ["packaging", "importlib-resources"]
+
 [[package]]
 category = "dev"
 description = "Various helpers to pass data to untrusted environments and back."
@@ -138,6 +211,32 @@ optional = false
 python-versions = "*"
 version = "0.82"
 
+[[package]]
+category = "main"
+description = "An implementation of JSON Schema validation for Python"
+name = "jsonschema"
+optional = false
+python-versions = "*"
+version = "3.2.0"
+
+[package.dependencies]
+attrs = ">=17.4.0"
+pyrsistent = ">=0.14.0"
+setuptools = "*"
+six = ">=1.11.0"
+
+[package.dependencies.functools32]
+python = "<3"
+version = "*"
+
+[package.dependencies.importlib-metadata]
+python = "<3.8"
+version = "*"
+
+[package.extras]
+format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"]
+format_nongpl = ["idna", "jsonpointer (>1.13)", "webcolors", "rfc3986-validator (>0.1.0)", "rfc3339-validator"]
+
 [[package]]
 category = "main"
 description = "Safely add untrusted strings to HTML/XML markup."
@@ -146,6 +245,45 @@ optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"
 version = "1.1.1"
 
+[[package]]
+category = "main"
+description = "More routines for operating on iterables, beyond itertools"
+marker = "python_version < \"3.8\""
+name = "more-itertools"
+optional = false
+python-versions = "*"
+version = "5.0.0"
+
+[package.dependencies]
+six = ">=1.0.0,<2.0.0"
+
+[[package]]
+category = "main"
+description = "Object-oriented filesystem paths"
+marker = "python_version < \"3\""
+name = "pathlib2"
+optional = false
+python-versions = "*"
+version = "2.3.5"
+
+[package.dependencies]
+six = "*"
+
+[package.dependencies.scandir]
+python = "<3.5"
+version = "*"
+
+[[package]]
+category = "main"
+description = "Persistent/Functional/Immutable data structures"
+name = "pyrsistent"
+optional = false
+python-versions = "*"
+version = "0.15.6"
+
+[package.dependencies]
+six = "*"
+
 [[package]]
 category = "main"
 description = "YAML parser and emitter for Python"
@@ -183,6 +321,15 @@ version = "0.9.1"
 [package.dependencies]
 requests = ">=2.0.1,<3.0.0"
 
+[[package]]
+category = "main"
+description = "scandir, a better directory iterator and faster os.walk()"
+marker = "python_version < \"3\""
+name = "scandir"
+optional = false
+python-versions = "*"
+version = "1.10.0"
+
 [[package]]
 category = "main"
 description = "Python client for Sentry (https://getsentry.com)"
@@ -209,6 +356,14 @@ sanic = ["sanic (>=0.8)"]
 sqlalchemy = ["sqlalchemy (>=1.2)"]
 tornado = ["tornado (>=5)"]
 
+[[package]]
+category = "main"
+description = "Python 2 and 3 compatibility utilities"
+name = "six"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*"
+version = "1.13.0"
+
 [[package]]
 category = "main"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
@@ -235,11 +390,31 @@ dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-i
 termcolor = ["termcolor"]
 watchdog = ["watchdog"]
 
+[[package]]
+category = "main"
+description = "Backport of pathlib-compatible object wrapper for zip files"
+marker = "python_version < \"3.8\""
+name = "zipp"
+optional = false
+python-versions = ">=2.7"
+version = "0.6.0"
+
+[package.dependencies]
+more-itertools = "*"
+
+[package.extras]
+docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
+testing = ["pathlib2", "contextlib2", "unittest2"]
+
 [metadata]
-content-hash = "7b478db27fe6f36aeed7f90b6c67efe5903fb43bb899bb66a1a65b80b8637c5a"
+content-hash = "843527171063a252e1b210f82037020d68f8aaed542c2d878e92d6c7e951a5f5"
 python-versions = "~2.7 || ^3.5"
 
 [metadata.files]
+attrs = [
+    {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"},
+    {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"},
+]
 certifi = [
     {file = "certifi-2019.11.28-py2.py3-none-any.whl", hash = "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3"},
     {file = "certifi-2019.11.28.tar.gz", hash = "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"},
@@ -260,6 +435,14 @@ colorlog = [
     {file = "colorlog-4.0.2-py2.py3-none-any.whl", hash = "sha256:450f52ea2a2b6ebb308f034ea9a9b15cea51e65650593dca1da3eb792e4e4981"},
     {file = "colorlog-4.0.2.tar.gz", hash = "sha256:3cf31b25cbc8f86ec01fef582ef3b840950dea414084ed19ab922c8b493f9b42"},
 ]
+configparser = [
+    {file = "configparser-4.0.2-py2.py3-none-any.whl", hash = "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c"},
+    {file = "configparser-4.0.2.tar.gz", hash = "sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df"},
+]
+contextlib2 = [
+    {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"},
+    {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"},
+]
 coverage = [
     {file = "coverage-4.5.4-cp26-cp26m-macosx_10_12_x86_64.whl", hash = "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28"},
     {file = "coverage-4.5.4-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c"},
@@ -308,6 +491,10 @@ flask = [
     {file = "Flask-0.12.4-py2.py3-none-any.whl", hash = "sha256:6c02dbaa5a9ef790d8219bdced392e2d549c10cd5a5ba4b6aa65126b2271af29"},
     {file = "Flask-0.12.4.tar.gz", hash = "sha256:2ea22336f6d388b4b242bc3abf8a01244a8aa3e236e7407469ef78c16ba355dd"},
 ]
+functools32 = [
+    {file = "functools32-3.2.3-2.tar.gz", hash = "sha256:f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d"},
+    {file = "functools32-3.2.3-2.zip", hash = "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0"},
+]
 future = [
     {file = "future-0.18.2.tar.gz", hash = "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"},
 ]
@@ -319,6 +506,10 @@ idna = [
     {file = "idna-2.8-py2.py3-none-any.whl", hash = "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"},
     {file = "idna-2.8.tar.gz", hash = "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407"},
 ]
+importlib-metadata = [
+    {file = "importlib_metadata-1.3.0-py2.py3-none-any.whl", hash = "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f"},
+    {file = "importlib_metadata-1.3.0.tar.gz", hash = "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45"},
+]
 itsdangerous = [
     {file = "itsdangerous-1.1.0-py2.py3-none-any.whl", hash = "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"},
     {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"},
@@ -330,6 +521,10 @@ jinja2 = [
 jsonpath = [
     {file = "jsonpath-0.82.tar.gz", hash = "sha256:46d3fd2016cd5b842283d547877a02c418a0fe9aa7a6b0ae344115a2c990fef4"},
 ]
+jsonschema = [
+    {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"},
+    {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"},
+]
 markupsafe = [
     {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"},
     {file = "MarkupSafe-1.1.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7"},
@@ -360,6 +555,18 @@ markupsafe = [
     {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"},
     {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
 ]
+more-itertools = [
+    {file = "more-itertools-5.0.0.tar.gz", hash = "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4"},
+    {file = "more_itertools-5.0.0-py2-none-any.whl", hash = "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc"},
+    {file = "more_itertools-5.0.0-py3-none-any.whl", hash = "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9"},
+]
+pathlib2 = [
+    {file = "pathlib2-2.3.5-py2.py3-none-any.whl", hash = "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db"},
+    {file = "pathlib2-2.3.5.tar.gz", hash = "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868"},
+]
+pyrsistent = [
+    {file = "pyrsistent-0.15.6.tar.gz", hash = "sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b"},
+]
 pyyaml = [
     {file = "PyYAML-5.2-cp27-cp27m-win32.whl", hash = "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc"},
     {file = "PyYAML-5.2-cp27-cp27m-win_amd64.whl", hash = "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4"},
@@ -381,10 +588,27 @@ requests-toolbelt = [
     {file = "requests-toolbelt-0.9.1.tar.gz", hash = "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"},
     {file = "requests_toolbelt-0.9.1-py2.py3-none-any.whl", hash = "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f"},
 ]
+scandir = [
+    {file = "scandir-1.10.0-cp27-cp27m-win32.whl", hash = "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188"},
+    {file = "scandir-1.10.0-cp27-cp27m-win_amd64.whl", hash = "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac"},
+    {file = "scandir-1.10.0-cp34-cp34m-win32.whl", hash = "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f"},
+    {file = "scandir-1.10.0-cp34-cp34m-win_amd64.whl", hash = "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e"},
+    {file = "scandir-1.10.0-cp35-cp35m-win32.whl", hash = "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f"},
+    {file = "scandir-1.10.0-cp35-cp35m-win_amd64.whl", hash = "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32"},
+    {file = "scandir-1.10.0-cp36-cp36m-win32.whl", hash = "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022"},
+    {file = "scandir-1.10.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4"},
+    {file = "scandir-1.10.0-cp37-cp37m-win32.whl", hash = "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173"},
+    {file = "scandir-1.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d"},
+    {file = "scandir-1.10.0.tar.gz", hash = "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae"},
+]
 sentry-sdk = [
     {file = "sentry-sdk-0.13.5.tar.gz", hash = "sha256:c6b919623e488134a728f16326c6f0bcdab7e3f59e7f4c472a90eea4d6d8fe82"},
     {file = "sentry_sdk-0.13.5-py2.py3-none-any.whl", hash = "sha256:05285942901d38c7ce2498aba50d8e87b361fc603281a5902dda98f3f8c5e145"},
 ]
+six = [
+    {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"},
+    {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"},
+]
 urllib3 = [
     {file = "urllib3-1.25.7-py2.py3-none-any.whl", hash = "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293"},
     {file = "urllib3-1.25.7.tar.gz", hash = "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"},
@@ -393,3 +617,7 @@ werkzeug = [
     {file = "Werkzeug-0.16.0-py2.py3-none-any.whl", hash = "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4"},
     {file = "Werkzeug-0.16.0.tar.gz", hash = "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7"},
 ]
+zipp = [
+    {file = "zipp-0.6.0-py2.py3-none-any.whl", hash = "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"},
+    {file = "zipp-0.6.0.tar.gz", hash = "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e"},
+]

pyproject.toml

@@ -43,6 +43,7 @@ jsonpath = "^0.82"
 sentry-sdk = "^0.13.5"
 future = { version = "^0.18.1", python = "~2.7" }
 enum34 = { version = "^1.1.6", python = "~2.7" }
+jsonschema = "^3.2.0"
 
 [tool.poetry.dev-dependencies]
 flask = "<1.0.0"

tests/test_loader/test_cases.py

@@ -279,6 +279,7 @@ class TestSuiteLoader(unittest.TestCase):
 
     def test_load_project_tests(self):
         buildup.load_project_data(os.path.join(os.getcwd(), "tests"))
-        api_file_path = os.path.join(os.getcwd(), "tests", "api", "get_token.yml")
+        self.assertIn("gen_md5", self.project_mapping["functions"])
-        self.assertIn(api_file_path, self.tests_def_mapping["api"])
         self.assertEqual(self.project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH")
+        self.assertEqual(self.project_mapping["PWD"], os.path.abspath(os.path.dirname(os.path.dirname(__file__))))
+        self.assertEqual(self.project_mapping["test_path"], os.path.abspath(os.path.dirname(os.path.dirname(__file__))))

tests/test_loader/test_check.py

@@ -5,16 +5,11 @@ from httprunner.loader import check
 
 class TestLoaderCheck(unittest.TestCase):
 
-    def test_is_function(self):
-        func = lambda x: x + 1
-        self.assertTrue(check.is_function(func))
-        self.assertTrue(check.is_function(check.is_testcase))
-
     def test_is_testcases(self):
         data_structure = "path/to/file"
-        self.assertFalse(check.is_testcases(data_structure))
+        self.assertFalse(check.is_test_content(data_structure))
         data_structure = ["path/to/file1", "path/to/file2"]
-        self.assertFalse(check.is_testcases(data_structure))
+        self.assertFalse(check.is_test_content(data_structure))
 
         data_structure = {
             "project_mapping": {
@@ -34,9 +29,12 @@ class TestLoaderCheck(unittest.TestCase):
                         {
                             'name': 'test step desc1',
                             'variables': [],  # optional
-                            'extract': [],  # optional
+                            'extract': {},  # optional
                             'validate': [],
-                            'request': {}
+                            'request': {
+                                "method": "GET",
+                                "url": "https://docs.httprunner.org"
+                            }
                         },
                         # test_dict2   # another test dict
                     ]
@@ -44,33 +42,4 @@ class TestLoaderCheck(unittest.TestCase):
                 # testcase_dict_2     # another testcase dict
             ]
         }
-        self.assertTrue(check.is_testcases(data_structure))
+        self.assertTrue(check.is_test_content(data_structure))
-        data_structure = [
-            {
-                "name": "desc1",
-                "config": {},
-                "api": {},
-                "testcases": ["testcase11", "testcase12"]
-            },
-            {
-                "name": "desc2",
-                "config": {},
-                "api": {},
-                "testcases": ["testcase21", "testcase22"]
-            }
-        ]
-        self.assertTrue(data_structure)
-
-    def test_is_variable(self):
-        var1 = 123
-        var2 = "abc"
-        self.assertTrue(check.is_variable(("var1", var1)))
-        self.assertTrue(check.is_variable(("var2", var2)))
-
-        __var = 123
-        self.assertFalse(check.is_variable(("__var", __var)))
-
-        func = lambda x: x + 1
-        self.assertFalse(check.is_variable(("func", func)))
-
-        self.assertFalse(check.is_variable(("unittest", unittest)))

tests/test_loader/test_load.py

@@ -3,6 +3,7 @@ import unittest
 
 from httprunner import exceptions
 from httprunner.loader import load
+from httprunner.loader.buildup import load_test_file
 
 
 class TestFileLoader(unittest.TestCase):
@@ -14,7 +15,7 @@ class TestFileLoader(unittest.TestCase):
             f.write("")
 
         with self.assertRaises(exceptions.FileFormatError):
-            load._load_yaml_file(yaml_tmp_file)
+            load_test_file(yaml_tmp_file)
 
         os.remove(yaml_tmp_file)
 
@@ -23,7 +24,7 @@ class TestFileLoader(unittest.TestCase):
             f.write("abc")
 
         with self.assertRaises(exceptions.FileFormatError):
-            load._load_yaml_file(yaml_tmp_file)
+            load_test_file(yaml_tmp_file)
 
         os.remove(yaml_tmp_file)
 
@@ -34,7 +35,7 @@ class TestFileLoader(unittest.TestCase):
             f.write("")
 
         with self.assertRaises(exceptions.FileFormatError):
-            load._load_json_file(json_tmp_file)
+            load_test_file(json_tmp_file)
 
         os.remove(json_tmp_file)
 
@@ -43,7 +44,7 @@ class TestFileLoader(unittest.TestCase):
             f.write("{}")
 
         with self.assertRaises(exceptions.FileFormatError):
-            load._load_json_file(json_tmp_file)
+            load_test_file(json_tmp_file)
 
         os.remove(json_tmp_file)
 
@@ -52,7 +53,7 @@ class TestFileLoader(unittest.TestCase):
             f.write("abc")
 
         with self.assertRaises(exceptions.FileFormatError):
-            load._load_json_file(json_tmp_file)
+            load_test_file(json_tmp_file)
 
         os.remove(json_tmp_file)