feat: add json schema validation for api

This commit is contained in:
debugtalk
2019-12-31 17:31:33 +08:00
parent e77f2bb189
commit f233e126b9
16 changed files with 617 additions and 326 deletions

View File

@@ -2,6 +2,7 @@ import importlib
import os
from httprunner import exceptions, logger, utils
from httprunner.loader.check import JsonSchemaCheck
from httprunner.loader.load import load_module_functions, load_file, load_dot_env_file, \
load_folder_files
from httprunner.loader.locate import init_project_working_directory, get_project_working_directory
@@ -356,7 +357,7 @@ def load_test_file(path):
elif "request" in raw_content:
# file_type: api
# TODO: add json schema validation for api
JsonSchemaCheck.check_api_format(raw_content)
loaded_content = raw_content
loaded_content["path"] = path
loaded_content["type"] = "api"

View File

@@ -1,10 +1,73 @@
import json
import os
import types
from httprunner import logger, exceptions
import jsonschema
from httprunner import exceptions, logger
schemas_root_dir = os.path.join(os.path.dirname(__file__), "schemas")
common_schema_path = os.path.join(schemas_root_dir, "common.schema.json")
api_schema_path = os.path.join(schemas_root_dir, "api.schema.json")
testcase_schema_v2_path = os.path.join(schemas_root_dir, "testcase.schema.v2.json")
testsuite_schema_v2_path = os.path.join(schemas_root_dir, "testsuite.schema.v2.json")
with open(api_schema_path) as f:
api_schema = json.load(f)
with open(common_schema_path) as f:
common_schema = json.load(f)
resolver = jsonschema.RefResolver("file://{}/".format(os.path.abspath(schemas_root_dir)), common_schema)
with open(testcase_schema_v2_path) as f:
testcase_schema_v2 = json.load(f)
with open(testsuite_schema_v2_path) as f:
testsuite_schema_v2 = json.load(f)
# TODO: validate data format with JSON schema
class JsonSchemaCheck(object):
@staticmethod
def check_api_format(content):
try:
jsonschema.validate(content, api_schema, resolver=resolver)
except jsonschema.exceptions.ValidationError as ex:
logger.log_error(str(ex))
raise exceptions.FileFormatError
return True
class JsonSchemaV1Check(JsonSchemaCheck):
pass
class JsonSchemaV2Check(JsonSchemaCheck):
@staticmethod
def check_testcase_format(content):
""" check testcase format v2 if valid
"""
try:
jsonschema.validate(content, testcase_schema_v2, resolver=resolver)
except jsonschema.exceptions.ValidationError as ex:
logger.log_error(str(ex))
raise exceptions.FileFormatError
return True
@staticmethod
def check_testsuite_format(content):
try:
jsonschema.validate(content, testsuite_schema_v2, resolver=resolver)
except jsonschema.exceptions.ValidationError as ex:
logger.log_error(str(ex))
raise exceptions.FileFormatError
return True
def is_testcase(data_structure):
""" check if data_structure is a testcase.
@@ -130,23 +193,6 @@ def is_testcase_path(path):
return True
def check_testcase_format(file_path, content):
""" check testcase format if valid
"""
# TODO: replace with JSON schema validation
if not content:
# testcase file content is empty
err_msg = u"Testcase file content is empty: {}".format(file_path)
logger.log_error(err_msg)
raise exceptions.FileFormatError(err_msg)
elif not isinstance(content, (list, dict)):
# testcase file content does not match testcase format
err_msg = u"Testcase file content format invalid: {}".format(file_path)
logger.log_error(err_msg)
raise exceptions.FileFormatError(err_msg)
def is_function(item):
""" Takes item object, returns True if it is a function.
"""

View File

@@ -7,7 +7,7 @@ import yaml
from httprunner import builtin
from httprunner import exceptions, logger, utils
from httprunner.loader.check import check_testcase_format, is_function
from httprunner.loader.check import is_function
from httprunner.loader.locate import get_project_working_directory
try:
@@ -28,7 +28,6 @@ def _load_yaml_file(yaml_file):
logger.log_error(str(ex))
raise exceptions.FileFormatError
check_testcase_format(yaml_file, yaml_content)
return yaml_content
@@ -43,7 +42,6 @@ def _load_json_file(json_file):
logger.log_error(err_msg)
raise exceptions.FileFormatError(err_msg)
check_testcase_format(json_file, json_content)
return json_content

View File

@@ -0,0 +1,35 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "httprunner api schema definition",
"type": "object",
"properties": {
"name": {
"$ref": "common.schema.json#/definitions/name"
},
"base_url": {
"$ref": "common.schema.json#/definitions/base_url"
},
"variables": {
"$ref": "common.schema.json#/definitions/variables"
},
"request": {
"$ref": "common.schema.json#/definitions/request"
},
"setup_hooks": {
"$ref": "common.schema.json#/definitions/hook"
},
"teardown_hooks": {
"$ref": "common.schema.json#/definitions/hook"
},
"extract": {
"$ref": "common.schema.json#/definitions/extract"
},
"validate": {
"$ref": "common.schema.json#/definitions/validate"
}
},
"required": [
"name",
"request"
]
}

View File

@@ -0,0 +1,196 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "common json schema definitions for httprunner api/testcase/testsuite",
"definitions": {
"name": {
"description": "used as api/teststep/testcase/testsuite identification",
"type": "string"
},
"base_url": {
"description": "The base_url will be used with relative URI",
"type": "string"
},
"variables": {
"description": "define variables for api/teststep/testcase/testsuite",
"type": "object"
},
"config": {
"description": "used in testcase/testsuite to configure common fields",
"type": "object",
"properties": {
"name": {
"$ref": "#/definitions/name"
},
"base_url": {
"$ref": "#/definitions/base_url"
},
"variables": {
"$ref": "#/definitions/variables"
},
"setup_hooks": {
"$ref": "#/definitions/hook"
},
"teardown_hooks": {
"$ref": "#/definitions/hook"
}
},
"required": ["name"]
},
"hook": {
"description": "used to define setup_hooks/teardown_hooks for api/teststep/testcase",
"type": "array",
"items": {
"type": "string"
}
},
"request": {
"description": "used to define a api request. properties is the same as python package `requests.request`",
"type": "object",
"properties": {
"method": {
"type": "string",
"description": "request method",
"enum": [
"GET",
"POST",
"OPTIONS",
"HEAD",
"PUT",
"PATCH",
"DELETE"
]
},
"url": {
"description": "request url, may be absolute or relative URI",
"type": "string"
},
"params": {
"description": "query string for request url",
"type": "object"
},
"data": {
"anyOf": [
{
"description": "request body in json format",
"type": "object"
},
{
"description": "request body in application/x-www-form-urlencoded format, e.g. a=1&b=2",
"type": "string"
},
{
"description": "request body in string format, e.g. '${prepare_data($a, $b)}'",
"type": "string"
}
]
},
"json": {
"description": "request body in json format",
"type": "object"
},
"headers": {
"description": "request headers",
"type": "object"
},
"cookies": {
"description": "request cookies",
"type": "object"
},
"files": {
"description": "request files, used to upload files",
"type": "object"
},
"auth": {
"description": "Auth tuple to enable Basic/Digest/Custom HTTP Auth.",
"type": "array"
},
"timeout": {
"description": "How many seconds to wait for the server to send data before giving up",
"type": "number"
},
"allow_redirects": {
"description": "Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to True",
"type": "boolean"
},
"proxies": {
"description": "Dictionary mapping protocol to the URL of the proxy",
"type": "object"
},
"verify": {
"oneOf": [
{
"description": "whether we verify the servers TLS certificate",
"type": "boolean"
},
{
"description": "path to a CA bundle to use",
"type": "string"
}
]
},
"stream": {
"description": "if False, the response content will be immediately downloaded.",
"type": "boolean"
},
"cert": {
"oneOf": [
{
"description": "path to ssl client cert file (.pem)",
"type": "string"
},
{
"description": "('cert', 'key') pair",
"type": "array",
"maxItems": 2,
"minItems": 2,
"items": {
"type": "string"
}
}
]
},
"upload": {
"description": "upload files",
"type": "object"
}
},
"required": [
"method",
"url"
]
},
"extract": {
"description": "used to extract session variables for later requests",
"type": "object",
"patternProperties": {
"^[A-Za-z_][A-Za-z0-9_]*$": {
"description": "extraction rule for session variable, maybe in jsonpath/regex/jmespath",
"type": "string"
}
}
},
"validate": {
"description": "used to validate response fields",
"type": "array",
"items": {
"description": "one validator definition",
"type": "object",
"propertyNames": {
"enum": [
"eq",
"lt",
"gt"
]
},
"patternProperties": {
"^[A-Za-z_][A-Za-z0-9_]*$": {
"description": "validate_func_name: [check_value, expect_value]",
"type": "array",
"minItems": 2,
"maxItems": 2
}
}
}
}
}
}

View File

@@ -1,42 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/api.schema.json",
"title": "Api for httprunner",
"description": "Api for httprunner",
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "Name of the api"
},
"base_url": {
"type": "string",
"description": "The base_url will be added before a relative URI"
},
"request": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/request.schema.json"
},
"variables": {
"type": "object",
"description": "Variables for the api"
},
"extract": {
"type": "array",
"description": "Extract rules",
"items": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json"
}
},
"validate": {
"type": "array",
"description": "Validate rules",
"items": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json"
}
}
},
"required": [
"name",
"request"
]
}

View File

@@ -1,25 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/config.schema.json",
"title": "Config for httprunner",
"description": "Used in teststep/testcase/testsuite",
"type": "object",
"properties": {
"name": {
"type": "string"
},
"base_url": {
"type": "string",
"description": "The base_url will be added before a relative URI"
},
"variables": {
"type": "object"
},
"setup_hooks": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json"
},
"teardown_hooks": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json"
}
}
}

View File

@@ -1,13 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json",
"title": "Extract rules for httprunner",
"description": "Used to extract variables for later requests",
"type": "object",
"patternProperties": {
".*": {
"description": "extracted_variable_name: extract_rule",
"type": "string"
}
}
}

View File

@@ -1,10 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json",
"title": "setup_hooks or teardown_hooks for httprunner",
"description": "Define setup_hooks or teardown_hooks for httprunner",
"type": "array",
"items": {
"type": "string"
}
}

View File

@@ -1,93 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/request.schema.json",
"title": "request for httprunner",
"description": "Used to define a api or used in a teststep. Same parameters as python's package 'requests'",
"type": "object",
"properties": {
"method": {
"type": "string",
"description": "Request method",
"enum": [
"GET",
"POST",
"OPTIONS",
"HEAD",
"PUT",
"PATCH",
"DELETE"
]
},
"url": {
"description": "Request url",
"type": "string"
},
"params": {
"type": "object",
"description": "Used to define the parameters of a 'GET' request"
},
"data": {
"type": "object",
"description": "Used to define the parameters of a 'POST' request in the application/x-www-form-urlencoded format"
},
"json": {
"type": "object",
"description": "Used to define the parameters of a 'POST' request in the application/json format"
},
"headers": {
"description": "Request headers",
"type": "object"
},
"cookies": {
"description": "Request cookies",
"type": "object"
},
"files": {
"type": "object"
},
"auth": {
"type": "array"
},
"timeout": {
"type": "number"
},
"allow_redirects": {
"type": "boolean"
},
"proxies": {
"type": "object"
},
"verify": {
"oneOf": [
{
"type": "boolean"
},
{
"type": "string"
}
]
},
"stream": {
"type": "boolean"
},
"cert": {
"oneOf": [
{
"type": "string"
},
{
"type": "array",
"maxItems": 2,
"minItems": 2,
"items": {
"type": "string"
}
}
]
}
},
"required": [
"method",
"url"
]
}

View File

@@ -1,15 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json",
"title": "Validate rule of httprunner",
"description": "Validate rule of httprunner",
"type": "object",
"patternProperties": {
".*": {
"type": "array",
"description": "validate_function_name: [check_value, expect_value]",
"minItems": 2,
"maxItems": 2
}
}
}

View File

@@ -0,0 +1,88 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "httprunner testcase schema definition",
"type": "object",
"properties": {
"config": {
"$ref": "common.schema.json#/definitions/config"
},
"teststeps": {
"description": "teststep of a testcase",
"type": "array",
"minItems": 1,
"items": {
"$ref": "#/definitions/teststep"
}
}
},
"required": [
"config",
"teststeps"
],
"definitions": {
"teststep": {
"type": "object",
"oneOf": [
{
"properties": {
"name": {
"$ref": "common.schema.json#/definitions/name"
},
"api": {
"description": "api reference, value is api file relative path",
"type": "string"
},
"variables": {
"$ref": "common.schema.json#/definitions/variables"
},
"extract": {
"$ref": "common.schema.json#/definitions/extract"
},
"validate": {
"$ref": "common.schema.json#/definitions/validate"
},
"setup_hooks": {
"$ref": "common.schema.json#/definitions/hook"
},
"teardown_hooks": {
"$ref": "common.schema.json#/definitions/hook"
}
},
"required": [
"name",
"api"
]
},
{
"properties": {
"name": {
"$ref": "common.schema.json#/definitions/name"
},
"request": {
"$ref": "common.schema.json#/definitions/request"
},
"variables": {
"$ref": "common.schema.json#/definitions/variables"
},
"extract": {
"$ref": "common.schema.json#/definitions/extract"
},
"validate": {
"$ref": "common.schema.json#/definitions/validate"
},
"setup_hooks": {
"$ref": "common.schema.json#/definitions/hook"
},
"teardown_hooks": {
"$ref": "common.schema.json#/definitions/hook"
}
},
"required": [
"name",
"request"
]
}
]
}
}
}

View File

@@ -1,104 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/v2/testcase.schema.json",
"title": "Testcase for httprunner",
"description": "Testcase for httprunner",
"type": "object",
"properties": {
"config": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/config.schema.json"
},
"teststeps": {
"description": "Teststep of a testcase",
"type": "array",
"items": {
"$ref": "#/definitions/teststep"
}
}
},
"required": [
"teststeps"
],
"definitions": {
"teststep": {
"type": "object",
"oneOf": [
{
"properties": {
"name": {
"description": "Teststep name",
"type": "string"
},
"api": {
"description": "Api reference, it's usually the relative path of the api",
"type": "string"
},
"variables": {
"type": "object"
},
"extract": {
"description": "Extract rules",
"type": "array",
"items": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json"
}
},
"validate": {
"description": "Validate rules",
"type": "array",
"items": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json"
}
},
"setup_hooks": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json"
},
"teardown_hooks": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json"
}
},
"required": [
"api"
]
},
{
"properties": {
"name": {
"description": "Teststep name",
"type": "string"
},
"request": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/request.schema.json"
},
"variables": {
"type": "object"
},
"extract": {
"description": "Extract rules",
"type": "array",
"items": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json"
}
},
"validate": {
"description": "Validate rules",
"type": "array",
"items": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json"
}
},
"setup_hooks": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json"
},
"teardown_hooks": {
"$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json"
}
},
"required": [
"request"
]
}
]
}
}
}