From 5483b49935c1d18d675872a2407aa16b8938a51f Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 30 Dec 2019 20:26:16 +0800 Subject: [PATCH 01/31] change: function name, validate_json_file => validate_test_file --- httprunner/cli.py | 4 ++-- httprunner/loader/__init__.py | 4 ++-- httprunner/loader/check.py | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/httprunner/cli.py b/httprunner/cli.py index 8258ba4f..5e8a5576 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -7,7 +7,7 @@ from sentry_sdk import capture_exception from httprunner import __description__, __version__ from httprunner.api import HttpRunner from httprunner.compat import is_py2 -from httprunner.loader import validate_json_file +from httprunner.loader import validate_test_file from httprunner.logger import color_print from httprunner.report import gen_html_report from httprunner.utils import (create_scaffold, get_python2_retire_msg, @@ -73,7 +73,7 @@ def main(): sys.exit(0) if args.validate: - validate_json_file(args.validate) + validate_test_file(args.validate) sys.exit(0) if args.prettify: prettify_json_file(args.prettify) diff --git a/httprunner/loader/__init__.py b/httprunner/loader/__init__.py index 46e18ad5..79fc4a3c 100644 --- a/httprunner/loader/__init__.py +++ b/httprunner/loader/__init__.py @@ -8,7 +8,7 @@ HttpRunner loader """ -from httprunner.loader.check import is_testcase_path, is_testcases, validate_json_file +from httprunner.loader.check import is_testcase_path, is_testcases, validate_test_file from httprunner.loader.locate import get_project_working_directory as get_pwd from httprunner.loader.load import load_csv_file, load_builtin_functions from httprunner.loader.buildup import load_cases, load_project_data @@ -16,7 +16,7 @@ from httprunner.loader.buildup import load_cases, load_project_data __all__ = [ "is_testcase_path", "is_testcases", - "validate_json_file", + "validate_test_file", "get_pwd", "load_csv_file", "load_builtin_functions", diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 9654709b..4e5943b3 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -149,17 +149,17 @@ def check_testcase_format(file_path, content): raise exceptions.FileFormatError(err_msg) -def validate_json_file(file_list): +def validate_test_file(file_list): """ validate JSON testcase format """ - for json_file in set(file_list): - if not json_file.endswith(".json"): - logger.log_warning("Only JSON file format can be validated, skip: {}".format(json_file)) + for test_file in set(file_list): + if not test_file.endswith(".json"): + logger.log_warning("Only JSON file format can be validated, skip: {}".format(test_file)) continue - logger.color_print("Start to validate JSON file: {}".format(json_file), "GREEN") + logger.color_print("Start to validate JSON file: {}".format(test_file), "GREEN") - with io.open(json_file) as stream: + with io.open(test_file) as stream: try: json.load(stream) except ValueError as e: From 3213f5bf5daeec643514ef63606fde8a222a4083 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 30 Dec 2019 21:50:36 +0800 Subject: [PATCH 02/31] refactor: use loader.load_cases to validate test files --- docs/CHANGELOG.md | 6 ++++++ httprunner/cli.py | 17 +++++++++++++---- httprunner/loader/__init__.py | 3 +-- httprunner/loader/check.py | 21 --------------------- httprunner/loader/load.py | 7 ++++++- 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8ed660b9..bff5a16a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 2.5.0 (2019-12-30) + +**Changed** + +- refactor: use loader.load_cases to validate test files + ## 2.4.9 (2019-12-29) **Added** diff --git a/httprunner/cli.py b/httprunner/cli.py index 5e8a5576..1314cb8a 100644 --- a/httprunner/cli.py +++ b/httprunner/cli.py @@ -4,11 +4,11 @@ import sys from sentry_sdk import capture_exception -from httprunner import __description__, __version__ +from httprunner import __description__, __version__, exceptions from httprunner.api import HttpRunner from httprunner.compat import is_py2 -from httprunner.loader import validate_test_file -from httprunner.logger import color_print +from httprunner.loader import load_cases +from httprunner.logger import color_print, log_error from httprunner.report import gen_html_report from httprunner.utils import (create_scaffold, get_python2_retire_msg, prettify_json_file) @@ -73,8 +73,17 @@ def main(): sys.exit(0) if args.validate: - validate_test_file(args.validate) + for validate_path in args.validate: + try: + color_print("validate test file: {}".format(validate_path), "GREEN") + load_cases(validate_path, args.dot_env_path) + except exceptions.MyBaseError as ex: + log_error(str(ex)) + continue + + color_print("done!", "BLUE") sys.exit(0) + if args.prettify: prettify_json_file(args.prettify) sys.exit(0) diff --git a/httprunner/loader/__init__.py b/httprunner/loader/__init__.py index 79fc4a3c..8c6320b2 100644 --- a/httprunner/loader/__init__.py +++ b/httprunner/loader/__init__.py @@ -8,7 +8,7 @@ HttpRunner loader """ -from httprunner.loader.check import is_testcase_path, is_testcases, validate_test_file +from httprunner.loader.check import is_testcase_path, is_testcases from httprunner.loader.locate import get_project_working_directory as get_pwd from httprunner.loader.load import load_csv_file, load_builtin_functions from httprunner.loader.buildup import load_cases, load_project_data @@ -16,7 +16,6 @@ from httprunner.loader.buildup import load_cases, load_project_data __all__ = [ "is_testcase_path", "is_testcases", - "validate_test_file", "get_pwd", "load_csv_file", "load_builtin_functions", diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 4e5943b3..3d535c9d 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -1,5 +1,3 @@ -import io -import json import os import types @@ -149,25 +147,6 @@ def check_testcase_format(file_path, content): raise exceptions.FileFormatError(err_msg) -def validate_test_file(file_list): - """ validate JSON testcase format - """ - for test_file in set(file_list): - if not test_file.endswith(".json"): - logger.log_warning("Only JSON file format can be validated, skip: {}".format(test_file)) - continue - - logger.color_print("Start to validate JSON file: {}".format(test_file), "GREEN") - - with io.open(test_file) as stream: - try: - json.load(stream) - except ValueError as e: - raise SystemExit(e) - - print("OK") - - def is_function(item): """ Takes item object, returns True if it is a function. """ diff --git a/httprunner/loader/load.py b/httprunner/loader/load.py index 3b783856..287b2560 100644 --- a/httprunner/loader/load.py +++ b/httprunner/loader/load.py @@ -22,7 +22,12 @@ def _load_yaml_file(yaml_file): """ load yaml file and check file content format """ with io.open(yaml_file, 'r', encoding='utf-8') as stream: - yaml_content = yaml.load(stream) + try: + yaml_content = yaml.load(stream) + except yaml.YAMLError as ex: + logger.log_error(str(ex)) + raise exceptions.FileFormatError + check_testcase_format(yaml_file, yaml_content) return yaml_content From 59dd35acff81695cb20e96c815d22f09117e2415 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 10:49:18 +0800 Subject: [PATCH 03/31] change: update json schemas v2 --- .../schemas/{v2 => common}/api.schema.json | 8 +++---- .../{v2 => }/common/config.schema.json | 6 ++--- .../{v2 => }/common/extract.schema.json | 2 +- .../schemas/{v2 => }/common/hook.schema.json | 2 +- .../{v2 => }/common/request.schema.json | 2 +- .../{v2 => }/common/validate.schema.json | 2 +- .../loader/schemas/v2/testcase.schema.json | 22 +++++++++---------- .../loader/schemas/v2/testsuite.schema.json | 8 +++---- 8 files changed, 26 insertions(+), 26 deletions(-) rename httprunner/loader/schemas/{v2 => common}/api.schema.json (62%) rename httprunner/loader/schemas/{v2 => }/common/config.schema.json (57%) rename httprunner/loader/schemas/{v2 => }/common/extract.schema.json (72%) rename httprunner/loader/schemas/{v2 => }/common/hook.schema.json (67%) rename httprunner/loader/schemas/{v2 => }/common/request.schema.json (94%) rename httprunner/loader/schemas/{v2 => }/common/validate.schema.json (75%) diff --git a/httprunner/loader/schemas/v2/api.schema.json b/httprunner/loader/schemas/common/api.schema.json similarity index 62% rename from httprunner/loader/schemas/v2/api.schema.json rename to httprunner/loader/schemas/common/api.schema.json index 431a61b8..e96096f3 100644 --- a/httprunner/loader/schemas/v2/api.schema.json +++ b/httprunner/loader/schemas/common/api.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/api.schema.json", + "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/api.schema.json", "title": "Api for httprunner", "description": "Api for httprunner", "type": "object", @@ -14,7 +14,7 @@ "description": "The base_url will be added before a relative URI" }, "request": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/request.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/request.schema.json" }, "variables": { "type": "object", @@ -24,14 +24,14 @@ "type": "array", "description": "Extract rules", "items": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/extract.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json" } }, "validate": { "type": "array", "description": "Validate rules", "items": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/validate.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json" } } }, diff --git a/httprunner/loader/schemas/v2/common/config.schema.json b/httprunner/loader/schemas/common/config.schema.json similarity index 57% rename from httprunner/loader/schemas/v2/common/config.schema.json rename to httprunner/loader/schemas/common/config.schema.json index 67de36ac..85026cdd 100644 --- a/httprunner/loader/schemas/v2/common/config.schema.json +++ b/httprunner/loader/schemas/common/config.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/config.schema.json", + "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/config.schema.json", "title": "Config for httprunner", "description": "Used in teststep/testcase/testsuite", "type": "object", @@ -16,10 +16,10 @@ "type": "object" }, "setup_hooks": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" }, "teardown_hooks": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" } } } \ No newline at end of file diff --git a/httprunner/loader/schemas/v2/common/extract.schema.json b/httprunner/loader/schemas/common/extract.schema.json similarity index 72% rename from httprunner/loader/schemas/v2/common/extract.schema.json rename to httprunner/loader/schemas/common/extract.schema.json index cf24863d..74d7fb5d 100644 --- a/httprunner/loader/schemas/v2/common/extract.schema.json +++ b/httprunner/loader/schemas/common/extract.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/extract.schema.json", + "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json", "title": "Extract rules for httprunner", "description": "Used to extract variables for later requests", "type": "object", diff --git a/httprunner/loader/schemas/v2/common/hook.schema.json b/httprunner/loader/schemas/common/hook.schema.json similarity index 67% rename from httprunner/loader/schemas/v2/common/hook.schema.json rename to httprunner/loader/schemas/common/hook.schema.json index f80be216..919b44b7 100644 --- a/httprunner/loader/schemas/v2/common/hook.schema.json +++ b/httprunner/loader/schemas/common/hook.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json", + "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json", "title": "setup_hooks or teardown_hooks for httprunner", "description": "Define setup_hooks or teardown_hooks for httprunner", "type": "array", diff --git a/httprunner/loader/schemas/v2/common/request.schema.json b/httprunner/loader/schemas/common/request.schema.json similarity index 94% rename from httprunner/loader/schemas/v2/common/request.schema.json rename to httprunner/loader/schemas/common/request.schema.json index c9385ea0..0cccb759 100644 --- a/httprunner/loader/schemas/v2/common/request.schema.json +++ b/httprunner/loader/schemas/common/request.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/request.schema.json", + "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/request.schema.json", "title": "request for httprunner", "description": "Used to define a api or used in a teststep. Same parameters as python's package 'requests'", "type": "object", diff --git a/httprunner/loader/schemas/v2/common/validate.schema.json b/httprunner/loader/schemas/common/validate.schema.json similarity index 75% rename from httprunner/loader/schemas/v2/common/validate.schema.json rename to httprunner/loader/schemas/common/validate.schema.json index 14371992..718a8d89 100644 --- a/httprunner/loader/schemas/v2/common/validate.schema.json +++ b/httprunner/loader/schemas/common/validate.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/validate.schema.json", + "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json", "title": "Validate rule of httprunner", "description": "Validate rule of httprunner", "type": "object", diff --git a/httprunner/loader/schemas/v2/testcase.schema.json b/httprunner/loader/schemas/v2/testcase.schema.json index b4c4198c..5490270e 100644 --- a/httprunner/loader/schemas/v2/testcase.schema.json +++ b/httprunner/loader/schemas/v2/testcase.schema.json @@ -1,12 +1,12 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/testcase.schema.json", + "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/v2/testcase.schema.json", "title": "Testcase for httprunner", "description": "Testcase for httprunner", "type": "object", "properties": { "config": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/config.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/config.schema.json" }, "teststeps": { "description": "Teststep of a testcase", @@ -40,21 +40,21 @@ "description": "Extract rules", "type": "array", "items": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/extract.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json" } }, "validate": { "description": "Validate rules", "type": "array", "items": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/validate.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json" } }, "setup_hooks": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" }, "teardown_hooks": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" } }, "required": [ @@ -68,7 +68,7 @@ "type": "string" }, "request": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/request.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/request.schema.json" }, "variables": { "type": "object" @@ -77,21 +77,21 @@ "description": "Extract rules", "type": "array", "items": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/extract.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json" } }, "validate": { "description": "Validate rules", "type": "array", "items": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/validate.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json" } }, "setup_hooks": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" }, "teardown_hooks": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" } }, "required": [ diff --git a/httprunner/loader/schemas/v2/testsuite.schema.json b/httprunner/loader/schemas/v2/testsuite.schema.json index 41e0de57..cfdfbb14 100644 --- a/httprunner/loader/schemas/v2/testsuite.schema.json +++ b/httprunner/loader/schemas/v2/testsuite.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/testsuite.schema.json", + "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/v2/testsuite.schema.json", "title": "Testsuite for httprunner", "description": "Testsuite for httprunner", "type": "object", @@ -8,7 +8,7 @@ "config": { "allOf": [ { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/config.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/config.schema.json" } ], "properties": { @@ -24,10 +24,10 @@ } }, "setup_hooks": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" }, "teardown_hooks": { - "$ref": "https://raw.githubusercontent.com/readyou/httprunner/dev/httprunner/loader/schemas/v2/common/hook.schema.json" + "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" } }, "required": [ From 7c44fac6d5396e0e72ce6719d7e29200f12b4e7b Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 17:31:33 +0800 Subject: [PATCH 04/31] feat: add json schema validation for api --- httprunner/loader/buildup.py | 3 +- httprunner/loader/check.py | 84 +++++-- httprunner/loader/load.py | 4 +- httprunner/loader/schemas/api.schema.json | 35 +++ httprunner/loader/schemas/common.schema.json | 196 +++++++++++++++ .../loader/schemas/common/api.schema.json | 42 ---- .../loader/schemas/common/config.schema.json | 25 -- .../loader/schemas/common/extract.schema.json | 13 - .../loader/schemas/common/hook.schema.json | 10 - .../loader/schemas/common/request.schema.json | 93 ------- .../schemas/common/validate.schema.json | 15 -- .../loader/schemas/testcase.schema.v2.json | 88 +++++++ ...e.schema.json => testsuite.schema.v2.json} | 0 .../loader/schemas/v2/testcase.schema.json | 104 -------- poetry.lock | 230 +++++++++++++++++- pyproject.toml | 1 + 16 files changed, 617 insertions(+), 326 deletions(-) create mode 100644 httprunner/loader/schemas/api.schema.json create mode 100644 httprunner/loader/schemas/common.schema.json delete mode 100644 httprunner/loader/schemas/common/api.schema.json delete mode 100644 httprunner/loader/schemas/common/config.schema.json delete mode 100644 httprunner/loader/schemas/common/extract.schema.json delete mode 100644 httprunner/loader/schemas/common/hook.schema.json delete mode 100644 httprunner/loader/schemas/common/request.schema.json delete mode 100644 httprunner/loader/schemas/common/validate.schema.json create mode 100644 httprunner/loader/schemas/testcase.schema.v2.json rename httprunner/loader/schemas/{v2/testsuite.schema.json => testsuite.schema.v2.json} (100%) delete mode 100644 httprunner/loader/schemas/v2/testcase.schema.json diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index b6839b93..7dd28edd 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -2,6 +2,7 @@ import importlib import os from httprunner import exceptions, logger, utils +from httprunner.loader.check import JsonSchemaCheck from httprunner.loader.load import load_module_functions, load_file, load_dot_env_file, \ load_folder_files from httprunner.loader.locate import init_project_working_directory, get_project_working_directory @@ -356,7 +357,7 @@ def load_test_file(path): elif "request" in raw_content: # file_type: api - # TODO: add json schema validation for api + JsonSchemaCheck.check_api_format(raw_content) loaded_content = raw_content loaded_content["path"] = path loaded_content["type"] = "api" diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 3d535c9d..e59d33e5 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -1,10 +1,73 @@ +import json import os import types -from httprunner import logger, exceptions +import jsonschema + +from httprunner import exceptions, logger + +schemas_root_dir = os.path.join(os.path.dirname(__file__), "schemas") +common_schema_path = os.path.join(schemas_root_dir, "common.schema.json") +api_schema_path = os.path.join(schemas_root_dir, "api.schema.json") +testcase_schema_v2_path = os.path.join(schemas_root_dir, "testcase.schema.v2.json") +testsuite_schema_v2_path = os.path.join(schemas_root_dir, "testsuite.schema.v2.json") + +with open(api_schema_path) as f: + api_schema = json.load(f) + +with open(common_schema_path) as f: + common_schema = json.load(f) + resolver = jsonschema.RefResolver("file://{}/".format(os.path.abspath(schemas_root_dir)), common_schema) + +with open(testcase_schema_v2_path) as f: + testcase_schema_v2 = json.load(f) + +with open(testsuite_schema_v2_path) as f: + testsuite_schema_v2 = json.load(f) -# TODO: validate data format with JSON schema +class JsonSchemaCheck(object): + + @staticmethod + def check_api_format(content): + + try: + jsonschema.validate(content, api_schema, resolver=resolver) + except jsonschema.exceptions.ValidationError as ex: + logger.log_error(str(ex)) + raise exceptions.FileFormatError + + return True + + +class JsonSchemaV1Check(JsonSchemaCheck): + pass + + +class JsonSchemaV2Check(JsonSchemaCheck): + + @staticmethod + def check_testcase_format(content): + """ check testcase format v2 if valid + """ + try: + jsonschema.validate(content, testcase_schema_v2, resolver=resolver) + except jsonschema.exceptions.ValidationError as ex: + logger.log_error(str(ex)) + raise exceptions.FileFormatError + + return True + + @staticmethod + def check_testsuite_format(content): + try: + jsonschema.validate(content, testsuite_schema_v2, resolver=resolver) + except jsonschema.exceptions.ValidationError as ex: + logger.log_error(str(ex)) + raise exceptions.FileFormatError + + return True + def is_testcase(data_structure): """ check if data_structure is a testcase. @@ -130,23 +193,6 @@ def is_testcase_path(path): return True -def check_testcase_format(file_path, content): - """ check testcase format if valid - """ - # TODO: replace with JSON schema validation - if not content: - # testcase file content is empty - err_msg = u"Testcase file content is empty: {}".format(file_path) - logger.log_error(err_msg) - raise exceptions.FileFormatError(err_msg) - - elif not isinstance(content, (list, dict)): - # testcase file content does not match testcase format - err_msg = u"Testcase file content format invalid: {}".format(file_path) - logger.log_error(err_msg) - raise exceptions.FileFormatError(err_msg) - - def is_function(item): """ Takes item object, returns True if it is a function. """ diff --git a/httprunner/loader/load.py b/httprunner/loader/load.py index 287b2560..bf75d20e 100644 --- a/httprunner/loader/load.py +++ b/httprunner/loader/load.py @@ -7,7 +7,7 @@ import yaml from httprunner import builtin from httprunner import exceptions, logger, utils -from httprunner.loader.check import check_testcase_format, is_function +from httprunner.loader.check import is_function from httprunner.loader.locate import get_project_working_directory try: @@ -28,7 +28,6 @@ def _load_yaml_file(yaml_file): logger.log_error(str(ex)) raise exceptions.FileFormatError - check_testcase_format(yaml_file, yaml_content) return yaml_content @@ -43,7 +42,6 @@ def _load_json_file(json_file): logger.log_error(err_msg) raise exceptions.FileFormatError(err_msg) - check_testcase_format(json_file, json_content) return json_content diff --git a/httprunner/loader/schemas/api.schema.json b/httprunner/loader/schemas/api.schema.json new file mode 100644 index 00000000..cd046fbc --- /dev/null +++ b/httprunner/loader/schemas/api.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "description": "httprunner api schema definition", + "type": "object", + "properties": { + "name": { + "$ref": "common.schema.json#/definitions/name" + }, + "base_url": { + "$ref": "common.schema.json#/definitions/base_url" + }, + "variables": { + "$ref": "common.schema.json#/definitions/variables" + }, + "request": { + "$ref": "common.schema.json#/definitions/request" + }, + "setup_hooks": { + "$ref": "common.schema.json#/definitions/hook" + }, + "teardown_hooks": { + "$ref": "common.schema.json#/definitions/hook" + }, + "extract": { + "$ref": "common.schema.json#/definitions/extract" + }, + "validate": { + "$ref": "common.schema.json#/definitions/validate" + } + }, + "required": [ + "name", + "request" + ] +} \ No newline at end of file diff --git a/httprunner/loader/schemas/common.schema.json b/httprunner/loader/schemas/common.schema.json new file mode 100644 index 00000000..027ee9dd --- /dev/null +++ b/httprunner/loader/schemas/common.schema.json @@ -0,0 +1,196 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "description": "common json schema definitions for httprunner api/testcase/testsuite", + "definitions": { + "name": { + "description": "used as api/teststep/testcase/testsuite identification", + "type": "string" + }, + "base_url": { + "description": "The base_url will be used with relative URI", + "type": "string" + }, + "variables": { + "description": "define variables for api/teststep/testcase/testsuite", + "type": "object" + }, + "config": { + "description": "used in testcase/testsuite to configure common fields", + "type": "object", + "properties": { + "name": { + "$ref": "#/definitions/name" + }, + "base_url": { + "$ref": "#/definitions/base_url" + }, + "variables": { + "$ref": "#/definitions/variables" + }, + "setup_hooks": { + "$ref": "#/definitions/hook" + }, + "teardown_hooks": { + "$ref": "#/definitions/hook" + } + }, + "required": ["name"] + }, + "hook": { + "description": "used to define setup_hooks/teardown_hooks for api/teststep/testcase", + "type": "array", + "items": { + "type": "string" + } + }, + "request": { + "description": "used to define a api request. properties is the same as python package `requests.request`", + "type": "object", + "properties": { + "method": { + "type": "string", + "description": "request method", + "enum": [ + "GET", + "POST", + "OPTIONS", + "HEAD", + "PUT", + "PATCH", + "DELETE" + ] + }, + "url": { + "description": "request url, may be absolute or relative URI", + "type": "string" + }, + "params": { + "description": "query string for request url", + "type": "object" + }, + "data": { + "anyOf": [ + { + "description": "request body in json format", + "type": "object" + }, + { + "description": "request body in application/x-www-form-urlencoded format, e.g. a=1&b=2", + "type": "string" + }, + { + "description": "request body in string format, e.g. '${prepare_data($a, $b)}'", + "type": "string" + } + ] + }, + "json": { + "description": "request body in json format", + "type": "object" + }, + "headers": { + "description": "request headers", + "type": "object" + }, + "cookies": { + "description": "request cookies", + "type": "object" + }, + "files": { + "description": "request files, used to upload files", + "type": "object" + }, + "auth": { + "description": "Auth tuple to enable Basic/Digest/Custom HTTP Auth.", + "type": "array" + }, + "timeout": { + "description": "How many seconds to wait for the server to send data before giving up", + "type": "number" + }, + "allow_redirects": { + "description": "Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to True", + "type": "boolean" + }, + "proxies": { + "description": "Dictionary mapping protocol to the URL of the proxy", + "type": "object" + }, + "verify": { + "oneOf": [ + { + "description": "whether we verify the server’s TLS certificate", + "type": "boolean" + }, + { + "description": "path to a CA bundle to use", + "type": "string" + } + ] + }, + "stream": { + "description": "if False, the response content will be immediately downloaded.", + "type": "boolean" + }, + "cert": { + "oneOf": [ + { + "description": "path to ssl client cert file (.pem)", + "type": "string" + }, + { + "description": "('cert', 'key') pair", + "type": "array", + "maxItems": 2, + "minItems": 2, + "items": { + "type": "string" + } + } + ] + }, + "upload": { + "description": "upload files", + "type": "object" + } + }, + "required": [ + "method", + "url" + ] + }, + "extract": { + "description": "used to extract session variables for later requests", + "type": "object", + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_]*$": { + "description": "extraction rule for session variable, maybe in jsonpath/regex/jmespath", + "type": "string" + } + } + }, + "validate": { + "description": "used to validate response fields", + "type": "array", + "items": { + "description": "one validator definition", + "type": "object", + "propertyNames": { + "enum": [ + "eq", + "lt", + "gt" + ] + }, + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_]*$": { + "description": "validate_func_name: [check_value, expect_value]", + "type": "array", + "minItems": 2, + "maxItems": 2 + } + } + } + } + } +} \ No newline at end of file diff --git a/httprunner/loader/schemas/common/api.schema.json b/httprunner/loader/schemas/common/api.schema.json deleted file mode 100644 index e96096f3..00000000 --- a/httprunner/loader/schemas/common/api.schema.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/api.schema.json", - "title": "Api for httprunner", - "description": "Api for httprunner", - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "Name of the api" - }, - "base_url": { - "type": "string", - "description": "The base_url will be added before a relative URI" - }, - "request": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/request.schema.json" - }, - "variables": { - "type": "object", - "description": "Variables for the api" - }, - "extract": { - "type": "array", - "description": "Extract rules", - "items": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json" - } - }, - "validate": { - "type": "array", - "description": "Validate rules", - "items": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json" - } - } - }, - "required": [ - "name", - "request" - ] -} \ No newline at end of file diff --git a/httprunner/loader/schemas/common/config.schema.json b/httprunner/loader/schemas/common/config.schema.json deleted file mode 100644 index 85026cdd..00000000 --- a/httprunner/loader/schemas/common/config.schema.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/config.schema.json", - "title": "Config for httprunner", - "description": "Used in teststep/testcase/testsuite", - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "base_url": { - "type": "string", - "description": "The base_url will be added before a relative URI" - }, - "variables": { - "type": "object" - }, - "setup_hooks": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" - }, - "teardown_hooks": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" - } - } -} \ No newline at end of file diff --git a/httprunner/loader/schemas/common/extract.schema.json b/httprunner/loader/schemas/common/extract.schema.json deleted file mode 100644 index 74d7fb5d..00000000 --- a/httprunner/loader/schemas/common/extract.schema.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json", - "title": "Extract rules for httprunner", - "description": "Used to extract variables for later requests", - "type": "object", - "patternProperties": { - ".*": { - "description": "extracted_variable_name: extract_rule", - "type": "string" - } - } -} \ No newline at end of file diff --git a/httprunner/loader/schemas/common/hook.schema.json b/httprunner/loader/schemas/common/hook.schema.json deleted file mode 100644 index 919b44b7..00000000 --- a/httprunner/loader/schemas/common/hook.schema.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json", - "title": "setup_hooks or teardown_hooks for httprunner", - "description": "Define setup_hooks or teardown_hooks for httprunner", - "type": "array", - "items": { - "type": "string" - } -} \ No newline at end of file diff --git a/httprunner/loader/schemas/common/request.schema.json b/httprunner/loader/schemas/common/request.schema.json deleted file mode 100644 index 0cccb759..00000000 --- a/httprunner/loader/schemas/common/request.schema.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/request.schema.json", - "title": "request for httprunner", - "description": "Used to define a api or used in a teststep. Same parameters as python's package 'requests'", - "type": "object", - "properties": { - "method": { - "type": "string", - "description": "Request method", - "enum": [ - "GET", - "POST", - "OPTIONS", - "HEAD", - "PUT", - "PATCH", - "DELETE" - ] - }, - "url": { - "description": "Request url", - "type": "string" - }, - "params": { - "type": "object", - "description": "Used to define the parameters of a 'GET' request" - }, - "data": { - "type": "object", - "description": "Used to define the parameters of a 'POST' request in the application/x-www-form-urlencoded format" - }, - "json": { - "type": "object", - "description": "Used to define the parameters of a 'POST' request in the application/json format" - }, - "headers": { - "description": "Request headers", - "type": "object" - }, - "cookies": { - "description": "Request cookies", - "type": "object" - }, - "files": { - "type": "object" - }, - "auth": { - "type": "array" - }, - "timeout": { - "type": "number" - }, - "allow_redirects": { - "type": "boolean" - }, - "proxies": { - "type": "object" - }, - "verify": { - "oneOf": [ - { - "type": "boolean" - }, - { - "type": "string" - } - ] - }, - "stream": { - "type": "boolean" - }, - "cert": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "array", - "maxItems": 2, - "minItems": 2, - "items": { - "type": "string" - } - } - ] - } - }, - "required": [ - "method", - "url" - ] -} \ No newline at end of file diff --git a/httprunner/loader/schemas/common/validate.schema.json b/httprunner/loader/schemas/common/validate.schema.json deleted file mode 100644 index 718a8d89..00000000 --- a/httprunner/loader/schemas/common/validate.schema.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json", - "title": "Validate rule of httprunner", - "description": "Validate rule of httprunner", - "type": "object", - "patternProperties": { - ".*": { - "type": "array", - "description": "validate_function_name: [check_value, expect_value]", - "minItems": 2, - "maxItems": 2 - } - } -} \ No newline at end of file diff --git a/httprunner/loader/schemas/testcase.schema.v2.json b/httprunner/loader/schemas/testcase.schema.v2.json new file mode 100644 index 00000000..42404866 --- /dev/null +++ b/httprunner/loader/schemas/testcase.schema.v2.json @@ -0,0 +1,88 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "description": "httprunner testcase schema definition", + "type": "object", + "properties": { + "config": { + "$ref": "common.schema.json#/definitions/config" + }, + "teststeps": { + "description": "teststep of a testcase", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/teststep" + } + } + }, + "required": [ + "config", + "teststeps" + ], + "definitions": { + "teststep": { + "type": "object", + "oneOf": [ + { + "properties": { + "name": { + "$ref": "common.schema.json#/definitions/name" + }, + "api": { + "description": "api reference, value is api file relative path", + "type": "string" + }, + "variables": { + "$ref": "common.schema.json#/definitions/variables" + }, + "extract": { + "$ref": "common.schema.json#/definitions/extract" + }, + "validate": { + "$ref": "common.schema.json#/definitions/validate" + }, + "setup_hooks": { + "$ref": "common.schema.json#/definitions/hook" + }, + "teardown_hooks": { + "$ref": "common.schema.json#/definitions/hook" + } + }, + "required": [ + "name", + "api" + ] + }, + { + "properties": { + "name": { + "$ref": "common.schema.json#/definitions/name" + }, + "request": { + "$ref": "common.schema.json#/definitions/request" + }, + "variables": { + "$ref": "common.schema.json#/definitions/variables" + }, + "extract": { + "$ref": "common.schema.json#/definitions/extract" + }, + "validate": { + "$ref": "common.schema.json#/definitions/validate" + }, + "setup_hooks": { + "$ref": "common.schema.json#/definitions/hook" + }, + "teardown_hooks": { + "$ref": "common.schema.json#/definitions/hook" + } + }, + "required": [ + "name", + "request" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/httprunner/loader/schemas/v2/testsuite.schema.json b/httprunner/loader/schemas/testsuite.schema.v2.json similarity index 100% rename from httprunner/loader/schemas/v2/testsuite.schema.json rename to httprunner/loader/schemas/testsuite.schema.v2.json diff --git a/httprunner/loader/schemas/v2/testcase.schema.json b/httprunner/loader/schemas/v2/testcase.schema.json deleted file mode 100644 index 5490270e..00000000 --- a/httprunner/loader/schemas/v2/testcase.schema.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/v2/testcase.schema.json", - "title": "Testcase for httprunner", - "description": "Testcase for httprunner", - "type": "object", - "properties": { - "config": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/config.schema.json" - }, - "teststeps": { - "description": "Teststep of a testcase", - "type": "array", - "items": { - "$ref": "#/definitions/teststep" - } - } - }, - "required": [ - "teststeps" - ], - "definitions": { - "teststep": { - "type": "object", - "oneOf": [ - { - "properties": { - "name": { - "description": "Teststep name", - "type": "string" - }, - "api": { - "description": "Api reference, it's usually the relative path of the api", - "type": "string" - }, - "variables": { - "type": "object" - }, - "extract": { - "description": "Extract rules", - "type": "array", - "items": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json" - } - }, - "validate": { - "description": "Validate rules", - "type": "array", - "items": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json" - } - }, - "setup_hooks": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" - }, - "teardown_hooks": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" - } - }, - "required": [ - "api" - ] - }, - { - "properties": { - "name": { - "description": "Teststep name", - "type": "string" - }, - "request": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/request.schema.json" - }, - "variables": { - "type": "object" - }, - "extract": { - "description": "Extract rules", - "type": "array", - "items": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/extract.schema.json" - } - }, - "validate": { - "description": "Validate rules", - "type": "array", - "items": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/validate.schema.json" - } - }, - "setup_hooks": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" - }, - "teardown_hooks": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" - } - }, - "required": [ - "request" - ] - } - ] - } - } -} \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index fe5355d3..f7070bd7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,17 @@ +[[package]] +category = "main" +description = "Classes Without Boilerplate" +name = "attrs" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "19.3.0" + +[package.extras] +azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] +dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"] +docs = ["sphinx", "zope.interface"] +tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] + [[package]] category = "main" description = "Python package for providing Mozilla's CA Bundle." @@ -41,6 +55,28 @@ version = "4.0.2" [package.dependencies] colorama = "*" +[[package]] +category = "main" +description = "Updated configparser from Python 3.7 for Python 2.6+." +marker = "python_version < \"3\"" +name = "configparser" +optional = false +python-versions = ">=2.6" +version = "4.0.2" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2)", "pytest-flake8", "pytest-black-multipy"] + +[[package]] +category = "main" +description = "Backports and enhancements for the contextlib module" +marker = "python_version < \"3\"" +name = "contextlib2" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.6.0.post1" + [[package]] category = "dev" description = "Code coverage measurement for Python" @@ -80,6 +116,15 @@ Werkzeug = ">=0.7" click = ">=2.0" itsdangerous = ">=0.21" +[[package]] +category = "main" +description = "Backport of the functools module from Python 3.2.3 for use on 2.7 and PyPy." +marker = "python_version < \"3\"" +name = "functools32" +optional = false +python-versions = "*" +version = "3.2.3-2" + [[package]] category = "main" description = "Clean single-source support for Python 3 and 2" @@ -108,6 +153,34 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "2.8" +[[package]] +category = "main" +description = "Read metadata from Python packages" +marker = "python_version < \"3.8\"" +name = "importlib-metadata" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +version = "1.3.0" + +[package.dependencies] +zipp = ">=0.5" + +[package.dependencies.configparser] +python = "<3" +version = ">=3.5" + +[package.dependencies.contextlib2] +python = "<3" +version = "*" + +[package.dependencies.pathlib2] +python = "<3" +version = "*" + +[package.extras] +docs = ["sphinx", "rst.linker"] +testing = ["packaging", "importlib-resources"] + [[package]] category = "dev" description = "Various helpers to pass data to untrusted environments and back." @@ -138,6 +211,32 @@ optional = false python-versions = "*" version = "0.82" +[[package]] +category = "main" +description = "An implementation of JSON Schema validation for Python" +name = "jsonschema" +optional = false +python-versions = "*" +version = "3.2.0" + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.dependencies.functools32] +python = "<3" +version = "*" + +[package.dependencies.importlib-metadata] +python = "<3.8" +version = "*" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format_nongpl = ["idna", "jsonpointer (>1.13)", "webcolors", "rfc3986-validator (>0.1.0)", "rfc3339-validator"] + [[package]] category = "main" description = "Safely add untrusted strings to HTML/XML markup." @@ -146,6 +245,45 @@ optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" version = "1.1.1" +[[package]] +category = "main" +description = "More routines for operating on iterables, beyond itertools" +marker = "python_version < \"3.8\"" +name = "more-itertools" +optional = false +python-versions = "*" +version = "5.0.0" + +[package.dependencies] +six = ">=1.0.0,<2.0.0" + +[[package]] +category = "main" +description = "Object-oriented filesystem paths" +marker = "python_version < \"3\"" +name = "pathlib2" +optional = false +python-versions = "*" +version = "2.3.5" + +[package.dependencies] +six = "*" + +[package.dependencies.scandir] +python = "<3.5" +version = "*" + +[[package]] +category = "main" +description = "Persistent/Functional/Immutable data structures" +name = "pyrsistent" +optional = false +python-versions = "*" +version = "0.15.6" + +[package.dependencies] +six = "*" + [[package]] category = "main" description = "YAML parser and emitter for Python" @@ -183,6 +321,15 @@ version = "0.9.1" [package.dependencies] requests = ">=2.0.1,<3.0.0" +[[package]] +category = "main" +description = "scandir, a better directory iterator and faster os.walk()" +marker = "python_version < \"3\"" +name = "scandir" +optional = false +python-versions = "*" +version = "1.10.0" + [[package]] category = "main" description = "Python client for Sentry (https://getsentry.com)" @@ -209,6 +356,14 @@ sanic = ["sanic (>=0.8)"] sqlalchemy = ["sqlalchemy (>=1.2)"] tornado = ["tornado (>=5)"] +[[package]] +category = "main" +description = "Python 2 and 3 compatibility utilities" +name = "six" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*" +version = "1.13.0" + [[package]] category = "main" description = "HTTP library with thread-safe connection pooling, file post, and more." @@ -235,11 +390,31 @@ dev = ["pytest", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-i termcolor = ["termcolor"] watchdog = ["watchdog"] +[[package]] +category = "main" +description = "Backport of pathlib-compatible object wrapper for zip files" +marker = "python_version < \"3.8\"" +name = "zipp" +optional = false +python-versions = ">=2.7" +version = "0.6.0" + +[package.dependencies] +more-itertools = "*" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pathlib2", "contextlib2", "unittest2"] + [metadata] -content-hash = "7b478db27fe6f36aeed7f90b6c67efe5903fb43bb899bb66a1a65b80b8637c5a" +content-hash = "843527171063a252e1b210f82037020d68f8aaed542c2d878e92d6c7e951a5f5" python-versions = "~2.7 || ^3.5" [metadata.files] +attrs = [ + {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, + {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, +] certifi = [ {file = "certifi-2019.11.28-py2.py3-none-any.whl", hash = "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3"}, {file = "certifi-2019.11.28.tar.gz", hash = "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"}, @@ -260,6 +435,14 @@ colorlog = [ {file = "colorlog-4.0.2-py2.py3-none-any.whl", hash = "sha256:450f52ea2a2b6ebb308f034ea9a9b15cea51e65650593dca1da3eb792e4e4981"}, {file = "colorlog-4.0.2.tar.gz", hash = "sha256:3cf31b25cbc8f86ec01fef582ef3b840950dea414084ed19ab922c8b493f9b42"}, ] +configparser = [ + {file = "configparser-4.0.2-py2.py3-none-any.whl", hash = "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c"}, + {file = "configparser-4.0.2.tar.gz", hash = "sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df"}, +] +contextlib2 = [ + {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"}, + {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, +] coverage = [ {file = "coverage-4.5.4-cp26-cp26m-macosx_10_12_x86_64.whl", hash = "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28"}, {file = "coverage-4.5.4-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c"}, @@ -308,6 +491,10 @@ flask = [ {file = "Flask-0.12.4-py2.py3-none-any.whl", hash = "sha256:6c02dbaa5a9ef790d8219bdced392e2d549c10cd5a5ba4b6aa65126b2271af29"}, {file = "Flask-0.12.4.tar.gz", hash = "sha256:2ea22336f6d388b4b242bc3abf8a01244a8aa3e236e7407469ef78c16ba355dd"}, ] +functools32 = [ + {file = "functools32-3.2.3-2.tar.gz", hash = "sha256:f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d"}, + {file = "functools32-3.2.3-2.zip", hash = "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0"}, +] future = [ {file = "future-0.18.2.tar.gz", hash = "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"}, ] @@ -319,6 +506,10 @@ idna = [ {file = "idna-2.8-py2.py3-none-any.whl", hash = "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"}, {file = "idna-2.8.tar.gz", hash = "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407"}, ] +importlib-metadata = [ + {file = "importlib_metadata-1.3.0-py2.py3-none-any.whl", hash = "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f"}, + {file = "importlib_metadata-1.3.0.tar.gz", hash = "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45"}, +] itsdangerous = [ {file = "itsdangerous-1.1.0-py2.py3-none-any.whl", hash = "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"}, {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"}, @@ -330,6 +521,10 @@ jinja2 = [ jsonpath = [ {file = "jsonpath-0.82.tar.gz", hash = "sha256:46d3fd2016cd5b842283d547877a02c418a0fe9aa7a6b0ae344115a2c990fef4"}, ] +jsonschema = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] markupsafe = [ {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"}, {file = "MarkupSafe-1.1.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7"}, @@ -360,6 +555,18 @@ markupsafe = [ {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"}, {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, ] +more-itertools = [ + {file = "more-itertools-5.0.0.tar.gz", hash = "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4"}, + {file = "more_itertools-5.0.0-py2-none-any.whl", hash = "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc"}, + {file = "more_itertools-5.0.0-py3-none-any.whl", hash = "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9"}, +] +pathlib2 = [ + {file = "pathlib2-2.3.5-py2.py3-none-any.whl", hash = "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db"}, + {file = "pathlib2-2.3.5.tar.gz", hash = "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868"}, +] +pyrsistent = [ + {file = "pyrsistent-0.15.6.tar.gz", hash = "sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b"}, +] pyyaml = [ {file = "PyYAML-5.2-cp27-cp27m-win32.whl", hash = "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc"}, {file = "PyYAML-5.2-cp27-cp27m-win_amd64.whl", hash = "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4"}, @@ -381,10 +588,27 @@ requests-toolbelt = [ {file = "requests-toolbelt-0.9.1.tar.gz", hash = "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"}, {file = "requests_toolbelt-0.9.1-py2.py3-none-any.whl", hash = "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f"}, ] +scandir = [ + {file = "scandir-1.10.0-cp27-cp27m-win32.whl", hash = "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188"}, + {file = "scandir-1.10.0-cp27-cp27m-win_amd64.whl", hash = "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac"}, + {file = "scandir-1.10.0-cp34-cp34m-win32.whl", hash = "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f"}, + {file = "scandir-1.10.0-cp34-cp34m-win_amd64.whl", hash = "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e"}, + {file = "scandir-1.10.0-cp35-cp35m-win32.whl", hash = "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f"}, + {file = "scandir-1.10.0-cp35-cp35m-win_amd64.whl", hash = "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32"}, + {file = "scandir-1.10.0-cp36-cp36m-win32.whl", hash = "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022"}, + {file = "scandir-1.10.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4"}, + {file = "scandir-1.10.0-cp37-cp37m-win32.whl", hash = "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173"}, + {file = "scandir-1.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d"}, + {file = "scandir-1.10.0.tar.gz", hash = "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae"}, +] sentry-sdk = [ {file = "sentry-sdk-0.13.5.tar.gz", hash = "sha256:c6b919623e488134a728f16326c6f0bcdab7e3f59e7f4c472a90eea4d6d8fe82"}, {file = "sentry_sdk-0.13.5-py2.py3-none-any.whl", hash = "sha256:05285942901d38c7ce2498aba50d8e87b361fc603281a5902dda98f3f8c5e145"}, ] +six = [ + {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"}, + {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"}, +] urllib3 = [ {file = "urllib3-1.25.7-py2.py3-none-any.whl", hash = "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293"}, {file = "urllib3-1.25.7.tar.gz", hash = "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"}, @@ -393,3 +617,7 @@ werkzeug = [ {file = "Werkzeug-0.16.0-py2.py3-none-any.whl", hash = "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4"}, {file = "Werkzeug-0.16.0.tar.gz", hash = "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7"}, ] +zipp = [ + {file = "zipp-0.6.0-py2.py3-none-any.whl", hash = "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"}, + {file = "zipp-0.6.0.tar.gz", hash = "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e"}, +] diff --git a/pyproject.toml b/pyproject.toml index 2dfa8519..1502485a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ jsonpath = "^0.82" sentry-sdk = "^0.13.5" future = { version = "^0.18.1", python = "~2.7" } enum34 = { version = "^1.1.6", python = "~2.7" } +jsonschema = "^3.2.0" [tool.poetry.dev-dependencies] flask = "<1.0.0" From 4aa2666ad4c8efde6bafe281eebb6c12c0bdabab Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 17:36:21 +0800 Subject: [PATCH 05/31] refactor: rename to JsonSchemaChecker --- httprunner/loader/buildup.py | 4 ++-- httprunner/loader/check.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index 7dd28edd..a7f0b09a 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -2,7 +2,7 @@ import importlib import os from httprunner import exceptions, logger, utils -from httprunner.loader.check import JsonSchemaCheck +from httprunner.loader.check import JsonSchemaChecker from httprunner.loader.load import load_module_functions, load_file, load_dot_env_file, \ load_folder_files from httprunner.loader.locate import init_project_working_directory, get_project_working_directory @@ -357,7 +357,7 @@ def load_test_file(path): elif "request" in raw_content: # file_type: api - JsonSchemaCheck.check_api_format(raw_content) + JsonSchemaChecker.validate_api_format(raw_content) loaded_content = raw_content loaded_content["path"] = path loaded_content["type"] = "api" diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index e59d33e5..6316e242 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -26,10 +26,10 @@ with open(testsuite_schema_v2_path) as f: testsuite_schema_v2 = json.load(f) -class JsonSchemaCheck(object): +class JsonSchemaChecker(object): @staticmethod - def check_api_format(content): + def validate_api_format(content): try: jsonschema.validate(content, api_schema, resolver=resolver) @@ -40,14 +40,14 @@ class JsonSchemaCheck(object): return True -class JsonSchemaV1Check(JsonSchemaCheck): +class JsonSchemaV1Checker(JsonSchemaChecker): pass -class JsonSchemaV2Check(JsonSchemaCheck): +class JsonSchemaV2Checker(JsonSchemaChecker): @staticmethod - def check_testcase_format(content): + def validate_testcase_format(content): """ check testcase format v2 if valid """ try: @@ -59,7 +59,7 @@ class JsonSchemaV2Check(JsonSchemaCheck): return True @staticmethod - def check_testsuite_format(content): + def validate_testsuite_format(content): try: jsonschema.validate(content, testsuite_schema_v2, resolver=resolver) except jsonschema.exceptions.ValidationError as ex: From 606b959d190fbca85f81a464788d1df01e145d80 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 17:49:15 +0800 Subject: [PATCH 06/31] fix: unittest for jsonschema checker --- httprunner/loader/__init__.py | 2 +- httprunner/loader/check.py | 2 -- tests/test_loader/test_load.py | 11 ++++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/httprunner/loader/__init__.py b/httprunner/loader/__init__.py index 8c6320b2..4b6ecccd 100644 --- a/httprunner/loader/__init__.py +++ b/httprunner/loader/__init__.py @@ -1,7 +1,7 @@ """ HttpRunner loader -- check: validate testcase data structure with JSON schema (TODO) +- check: validate api/testcase/testsuite data structure with JSON schema - locate: locate debugtalk.py, make it's dir as project root path - load: load testcase files and relevant data, including debugtalk.py, .env, yaml/json api/testcases, csv, etc. - buildup: assemble loaded content to httprunner testcase/testsuite data structure diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 6316e242..c73156b6 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -188,8 +188,6 @@ def is_testcase_path(path): if not os.path.exists(path): return False - # TODO: check file format if valid - return True diff --git a/tests/test_loader/test_load.py b/tests/test_loader/test_load.py index b394032e..4a2e0f91 100644 --- a/tests/test_loader/test_load.py +++ b/tests/test_loader/test_load.py @@ -3,6 +3,7 @@ import unittest from httprunner import exceptions from httprunner.loader import load +from httprunner.loader.buildup import load_test_file class TestFileLoader(unittest.TestCase): @@ -14,7 +15,7 @@ class TestFileLoader(unittest.TestCase): f.write("") with self.assertRaises(exceptions.FileFormatError): - load._load_yaml_file(yaml_tmp_file) + load_test_file(yaml_tmp_file) os.remove(yaml_tmp_file) @@ -23,7 +24,7 @@ class TestFileLoader(unittest.TestCase): f.write("abc") with self.assertRaises(exceptions.FileFormatError): - load._load_yaml_file(yaml_tmp_file) + load_test_file(yaml_tmp_file) os.remove(yaml_tmp_file) @@ -34,7 +35,7 @@ class TestFileLoader(unittest.TestCase): f.write("") with self.assertRaises(exceptions.FileFormatError): - load._load_json_file(json_tmp_file) + load_test_file(json_tmp_file) os.remove(json_tmp_file) @@ -43,7 +44,7 @@ class TestFileLoader(unittest.TestCase): f.write("{}") with self.assertRaises(exceptions.FileFormatError): - load._load_json_file(json_tmp_file) + load_test_file(json_tmp_file) os.remove(json_tmp_file) @@ -52,7 +53,7 @@ class TestFileLoader(unittest.TestCase): f.write("abc") with self.assertRaises(exceptions.FileFormatError): - load._load_json_file(json_tmp_file) + load_test_file(json_tmp_file) os.remove(json_tmp_file) From 36dd21fe2b896567681017e6fd87f36b5fce25dd Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 17:50:15 +0800 Subject: [PATCH 07/31] doc: update changelog --- docs/CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bff5a16a..462184f0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,10 @@ # Release History -## 2.5.0 (2019-12-30) +## 2.5.0 (2019-12-31) + +**Added** + +- feat: add json schema validation for api **Changed** From 4f06ca3cd1963584938b62b66891365b84db06e8 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 18:27:58 +0800 Subject: [PATCH 08/31] feat: add json schema validation for testcase v2 --- docs/CHANGELOG.md | 1 + httprunner/loader/buildup.py | 3 +- .../loader/schemas/testcase.schema.v2.json | 36 +++++++++---------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 462184f0..362cedb7 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,7 @@ **Added** - feat: add json schema validation for api +- feat: add json schema validation for testcase v2 **Changed** diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index a7f0b09a..0e95911a 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -2,7 +2,7 @@ import importlib import os from httprunner import exceptions, logger, utils -from httprunner.loader.check import JsonSchemaChecker +from httprunner.loader.check import JsonSchemaChecker, JsonSchemaV2Checker from httprunner.loader.load import load_module_functions, load_file, load_dot_env_file, \ load_folder_files from httprunner.loader.locate import init_project_working_directory, get_project_working_directory @@ -225,6 +225,7 @@ def load_testcase_v2(raw_testcase): } """ + JsonSchemaV2Checker.validate_testcase_format(raw_testcase) raw_teststeps = raw_testcase.pop("teststeps") raw_testcase["teststeps"] = [ load_teststep(teststep) diff --git a/httprunner/loader/schemas/testcase.schema.v2.json b/httprunner/loader/schemas/testcase.schema.v2.json index 42404866..09729489 100644 --- a/httprunner/loader/schemas/testcase.schema.v2.json +++ b/httprunner/loader/schemas/testcase.schema.v2.json @@ -2,23 +2,6 @@ "$schema": "http://json-schema.org/draft-07/schema", "description": "httprunner testcase schema definition", "type": "object", - "properties": { - "config": { - "$ref": "common.schema.json#/definitions/config" - }, - "teststeps": { - "description": "teststep of a testcase", - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/teststep" - } - } - }, - "required": [ - "config", - "teststeps" - ], "definitions": { "teststep": { "type": "object", @@ -84,5 +67,22 @@ } ] } - } + }, + "properties": { + "config": { + "$ref": "common.schema.json#/definitions/config" + }, + "teststeps": { + "description": "teststep of a testcase", + "type": "array", + "minItems": 1, + "items": { + "$ref": "testcase.schema.v2.json#/definitions/teststep" + } + } + }, + "required": [ + "config", + "teststeps" + ] } \ No newline at end of file From a2c2b5984a865bf86d1cc4a0f6cd13458ecae35f Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 18:32:04 +0800 Subject: [PATCH 09/31] change: add verify in config json schema --- httprunner/loader/schemas/common.schema.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/httprunner/loader/schemas/common.schema.json b/httprunner/loader/schemas/common.schema.json index 027ee9dd..09863f7c 100644 --- a/httprunner/loader/schemas/common.schema.json +++ b/httprunner/loader/schemas/common.schema.json @@ -32,6 +32,19 @@ }, "teardown_hooks": { "$ref": "#/definitions/hook" + }, + "verify": { + "description": "configure verify for current testcase/testsuite", + "oneOf": [ + { + "description": "whether we verify the server’s TLS certificate", + "type": "boolean" + }, + { + "description": "path to a CA bundle to use", + "type": "string" + } + ] } }, "required": ["name"] @@ -117,6 +130,7 @@ "type": "object" }, "verify": { + "description": "configure verify for current api/teststep", "oneOf": [ { "description": "whether we verify the server’s TLS certificate", From c334c9f33520abe5efff0d53c6690d3a548e9e17 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 18:57:20 +0800 Subject: [PATCH 10/31] feat: add json schema validation for testsuite --- docs/CHANGELOG.md | 1 + httprunner/loader/buildup.py | 1 + httprunner/loader/check.py | 26 ++++----- .../loader/schemas/testcase.schema.v2.json | 2 +- .../loader/schemas/testsuite.schema.json | 46 +++++++++++++++ .../loader/schemas/testsuite.schema.v2.json | 58 ------------------- 6 files changed, 62 insertions(+), 72 deletions(-) create mode 100644 httprunner/loader/schemas/testsuite.schema.json delete mode 100644 httprunner/loader/schemas/testsuite.schema.v2.json diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 362cedb7..3e6f92b6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,7 @@ - feat: add json schema validation for api - feat: add json schema validation for testcase v2 +- feat: add json schema validation for testsuite **Changed** diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index 0e95911a..01361880 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -281,6 +281,7 @@ def load_testsuite(raw_testsuite): } """ + JsonSchemaChecker.validate_testsuite_format(raw_testsuite) raw_testcases = raw_testsuite.pop("testcases") raw_testsuite["testcases"] = {} diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index c73156b6..7f389e25 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -10,7 +10,7 @@ schemas_root_dir = os.path.join(os.path.dirname(__file__), "schemas") common_schema_path = os.path.join(schemas_root_dir, "common.schema.json") api_schema_path = os.path.join(schemas_root_dir, "api.schema.json") testcase_schema_v2_path = os.path.join(schemas_root_dir, "testcase.schema.v2.json") -testsuite_schema_v2_path = os.path.join(schemas_root_dir, "testsuite.schema.v2.json") +testsuite_schema_path = os.path.join(schemas_root_dir, "testsuite.schema.json") with open(api_schema_path) as f: api_schema = json.load(f) @@ -22,8 +22,8 @@ with open(common_schema_path) as f: with open(testcase_schema_v2_path) as f: testcase_schema_v2 = json.load(f) -with open(testsuite_schema_v2_path) as f: - testsuite_schema_v2 = json.load(f) +with open(testsuite_schema_path) as f: + testsuite_schema = json.load(f) class JsonSchemaChecker(object): @@ -39,6 +39,16 @@ class JsonSchemaChecker(object): return True + @staticmethod + def validate_testsuite_format(content): + try: + jsonschema.validate(content, testsuite_schema, resolver=resolver) + except jsonschema.exceptions.ValidationError as ex: + logger.log_error(str(ex)) + raise exceptions.FileFormatError + + return True + class JsonSchemaV1Checker(JsonSchemaChecker): pass @@ -58,16 +68,6 @@ class JsonSchemaV2Checker(JsonSchemaChecker): return True - @staticmethod - def validate_testsuite_format(content): - try: - jsonschema.validate(content, testsuite_schema_v2, resolver=resolver) - except jsonschema.exceptions.ValidationError as ex: - logger.log_error(str(ex)) - raise exceptions.FileFormatError - - return True - def is_testcase(data_structure): """ check if data_structure is a testcase. diff --git a/httprunner/loader/schemas/testcase.schema.v2.json b/httprunner/loader/schemas/testcase.schema.v2.json index 09729489..4a1f2395 100644 --- a/httprunner/loader/schemas/testcase.schema.v2.json +++ b/httprunner/loader/schemas/testcase.schema.v2.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "description": "httprunner testcase schema definition", + "description": "httprunner testcase schema v2 definition", "type": "object", "definitions": { "teststep": { diff --git a/httprunner/loader/schemas/testsuite.schema.json b/httprunner/loader/schemas/testsuite.schema.json new file mode 100644 index 00000000..6d021123 --- /dev/null +++ b/httprunner/loader/schemas/testsuite.schema.json @@ -0,0 +1,46 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "description": "httprunner testsuite schema v2 definition", + "type": "object", + "definitions": { + "testcase": { + "type": "object", + "properties": { + "name": { + "$ref": "common.schema.json#/definitions/name" + }, + "variables": { + "$ref": "common.schema.json#/definitions/variables" + }, + "parameters": { + "description": "generate cartesian product variables with parameters, each group of variables will be run once", + "type": "object" + }, + "testcase": { + "description": "testcase reference, value is testcase file relative path", + "type": "string" + } + }, + "required": [ + "testcase" + ] + } + }, + "properties": { + "config": { + "$ref": "common.schema.json#/definitions/config" + }, + "testcases": { + "description": "testcase of a testsuite", + "type": "array", + "minItems": 1, + "items": { + "$ref": "testsuite.schema.json#/definitions/testcase" + } + } + }, + "required": [ + "config", + "testcases" + ] +} \ No newline at end of file diff --git a/httprunner/loader/schemas/testsuite.schema.v2.json b/httprunner/loader/schemas/testsuite.schema.v2.json deleted file mode 100644 index cfdfbb14..00000000 --- a/httprunner/loader/schemas/testsuite.schema.v2.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/v2/testsuite.schema.json", - "title": "Testsuite for httprunner", - "description": "Testsuite for httprunner", - "type": "object", - "properties": { - "config": { - "allOf": [ - { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/config.schema.json" - } - ], - "properties": { - "verify": { - "type": "boolean" - } - } - }, - "testcases": { - "type": "array", - "items": { - "$ref": "#/definitions/testcase" - } - }, - "setup_hooks": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" - }, - "teardown_hooks": { - "$ref": "https://raw.githubusercontent.com/httprunner/httprunner/master/httprunner/loader/schemas/common/hook.schema.json" - } - }, - "required": [ - "testcases" - ], - "definitions": { - "testcase": { - "type": "object", - "properties": { - "name": { - "description": "Testcase name", - "type": "string" - }, - "parameters": { - "description": "Parameters will generate cartesian product variables, each set of variables is tested once", - "type": "object" - }, - "testcase": { - "description": "Testcase reference, it's usually the relative path of the testcase", - "type": "string" - } - }, - "required": [ - "testcase" - ] - } - } -} \ No newline at end of file From 89c5d2e227ec6938229e63018ad04d783c3b0b54 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 18:59:46 +0800 Subject: [PATCH 11/31] refactor: rename to JsonSchemaCommonChecker --- httprunner/loader/buildup.py | 6 +++--- httprunner/loader/check.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index 01361880..35d9ebd0 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -2,7 +2,7 @@ import importlib import os from httprunner import exceptions, logger, utils -from httprunner.loader.check import JsonSchemaChecker, JsonSchemaV2Checker +from httprunner.loader.check import JsonSchemaCommonChecker, JsonSchemaV2Checker from httprunner.loader.load import load_module_functions, load_file, load_dot_env_file, \ load_folder_files from httprunner.loader.locate import init_project_working_directory, get_project_working_directory @@ -281,7 +281,7 @@ def load_testsuite(raw_testsuite): } """ - JsonSchemaChecker.validate_testsuite_format(raw_testsuite) + JsonSchemaCommonChecker.validate_testsuite_format(raw_testsuite) raw_testcases = raw_testsuite.pop("testcases") raw_testsuite["testcases"] = {} @@ -359,7 +359,7 @@ def load_test_file(path): elif "request" in raw_content: # file_type: api - JsonSchemaChecker.validate_api_format(raw_content) + JsonSchemaCommonChecker.validate_api_format(raw_content) loaded_content = raw_content loaded_content["path"] = path loaded_content["type"] = "api" diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 7f389e25..314bb548 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -26,7 +26,7 @@ with open(testsuite_schema_path) as f: testsuite_schema = json.load(f) -class JsonSchemaChecker(object): +class JsonSchemaCommonChecker(object): @staticmethod def validate_api_format(content): @@ -50,11 +50,11 @@ class JsonSchemaChecker(object): return True -class JsonSchemaV1Checker(JsonSchemaChecker): +class JsonSchemaV1Checker(JsonSchemaCommonChecker): pass -class JsonSchemaV2Checker(JsonSchemaChecker): +class JsonSchemaV2Checker(JsonSchemaCommonChecker): @staticmethod def validate_testcase_format(content): From af1de8ed1a65cd836052eafb302a89ff5fcd0f06 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 19:03:34 +0800 Subject: [PATCH 12/31] refactor: rename to JsonSchemaTestcaseChecker --- httprunner/loader/buildup.py | 4 ++-- httprunner/loader/check.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index 35d9ebd0..2a0fd08f 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -2,7 +2,7 @@ import importlib import os from httprunner import exceptions, logger, utils -from httprunner.loader.check import JsonSchemaCommonChecker, JsonSchemaV2Checker +from httprunner.loader.check import JsonSchemaCommonChecker, JsonSchemaTestcaseChecker from httprunner.loader.load import load_module_functions, load_file, load_dot_env_file, \ load_folder_files from httprunner.loader.locate import init_project_working_directory, get_project_working_directory @@ -225,7 +225,7 @@ def load_testcase_v2(raw_testcase): } """ - JsonSchemaV2Checker.validate_testcase_format(raw_testcase) + JsonSchemaTestcaseChecker.validate_testcase_v2_format(raw_testcase) raw_teststeps = raw_testcase.pop("teststeps") raw_testcase["teststeps"] = [ load_teststep(teststep) diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 314bb548..60d0db86 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -50,14 +50,16 @@ class JsonSchemaCommonChecker(object): return True -class JsonSchemaV1Checker(JsonSchemaCommonChecker): - pass - - -class JsonSchemaV2Checker(JsonSchemaCommonChecker): +class JsonSchemaTestcaseChecker(JsonSchemaCommonChecker): @staticmethod - def validate_testcase_format(content): + def validate_testcase_v1_format(content): + """ check testcase format v1 if valid + """ + return True + + @staticmethod + def validate_testcase_v2_format(content): """ check testcase format v2 if valid """ try: From 6d5943c0f3922f022cb74a049e9f985b6ad39a00 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 19:09:35 +0800 Subject: [PATCH 13/31] refactor: rename to JsonSchemaChecker --- httprunner/loader/buildup.py | 8 ++++---- httprunner/loader/check.py | 5 +---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index 2a0fd08f..085f3462 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -2,7 +2,7 @@ import importlib import os from httprunner import exceptions, logger, utils -from httprunner.loader.check import JsonSchemaCommonChecker, JsonSchemaTestcaseChecker +from httprunner.loader.check import JsonSchemaChecker from httprunner.loader.load import load_module_functions, load_file, load_dot_env_file, \ load_folder_files from httprunner.loader.locate import init_project_working_directory, get_project_working_directory @@ -225,7 +225,7 @@ def load_testcase_v2(raw_testcase): } """ - JsonSchemaTestcaseChecker.validate_testcase_v2_format(raw_testcase) + JsonSchemaChecker.validate_testcase_v2_format(raw_testcase) raw_teststeps = raw_testcase.pop("teststeps") raw_testcase["teststeps"] = [ load_teststep(teststep) @@ -281,7 +281,7 @@ def load_testsuite(raw_testsuite): } """ - JsonSchemaCommonChecker.validate_testsuite_format(raw_testsuite) + JsonSchemaChecker.validate_testsuite_format(raw_testsuite) raw_testcases = raw_testsuite.pop("testcases") raw_testsuite["testcases"] = {} @@ -359,7 +359,7 @@ def load_test_file(path): elif "request" in raw_content: # file_type: api - JsonSchemaCommonChecker.validate_api_format(raw_content) + JsonSchemaChecker.validate_api_format(raw_content) loaded_content = raw_content loaded_content["path"] = path loaded_content["type"] = "api" diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 60d0db86..f8feff26 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -26,7 +26,7 @@ with open(testsuite_schema_path) as f: testsuite_schema = json.load(f) -class JsonSchemaCommonChecker(object): +class JsonSchemaChecker(object): @staticmethod def validate_api_format(content): @@ -49,9 +49,6 @@ class JsonSchemaCommonChecker(object): return True - -class JsonSchemaTestcaseChecker(JsonSchemaCommonChecker): - @staticmethod def validate_testcase_v1_format(content): """ check testcase format v1 if valid From fffb2da7f675b3218c78fb26d55b4055b617d31f Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 19:11:21 +0800 Subject: [PATCH 14/31] change: add json schema validation for testsuite v2 --- docs/CHANGELOG.md | 2 +- httprunner/loader/check.py | 8 ++++---- .../{testsuite.schema.json => testsuite.schema.v2.json} | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename httprunner/loader/schemas/{testsuite.schema.json => testsuite.schema.v2.json} (94%) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 3e6f92b6..a193c99f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,7 +6,7 @@ - feat: add json schema validation for api - feat: add json schema validation for testcase v2 -- feat: add json schema validation for testsuite +- feat: add json schema validation for testsuite v2 **Changed** diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index f8feff26..853d1d59 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -10,7 +10,7 @@ schemas_root_dir = os.path.join(os.path.dirname(__file__), "schemas") common_schema_path = os.path.join(schemas_root_dir, "common.schema.json") api_schema_path = os.path.join(schemas_root_dir, "api.schema.json") testcase_schema_v2_path = os.path.join(schemas_root_dir, "testcase.schema.v2.json") -testsuite_schema_path = os.path.join(schemas_root_dir, "testsuite.schema.json") +testsuite_schema_v2_path = os.path.join(schemas_root_dir, "testsuite.schema.v2.json") with open(api_schema_path) as f: api_schema = json.load(f) @@ -22,8 +22,8 @@ with open(common_schema_path) as f: with open(testcase_schema_v2_path) as f: testcase_schema_v2 = json.load(f) -with open(testsuite_schema_path) as f: - testsuite_schema = json.load(f) +with open(testsuite_schema_v2_path) as f: + testsuite_schema_v2 = json.load(f) class JsonSchemaChecker(object): @@ -42,7 +42,7 @@ class JsonSchemaChecker(object): @staticmethod def validate_testsuite_format(content): try: - jsonschema.validate(content, testsuite_schema, resolver=resolver) + jsonschema.validate(content, testsuite_schema_v2, resolver=resolver) except jsonschema.exceptions.ValidationError as ex: logger.log_error(str(ex)) raise exceptions.FileFormatError diff --git a/httprunner/loader/schemas/testsuite.schema.json b/httprunner/loader/schemas/testsuite.schema.v2.json similarity index 94% rename from httprunner/loader/schemas/testsuite.schema.json rename to httprunner/loader/schemas/testsuite.schema.v2.json index 6d021123..f4dd1f5a 100644 --- a/httprunner/loader/schemas/testsuite.schema.json +++ b/httprunner/loader/schemas/testsuite.schema.v2.json @@ -35,7 +35,7 @@ "type": "array", "minItems": 1, "items": { - "$ref": "testsuite.schema.json#/definitions/testcase" + "$ref": "testsuite.schema.v2.json#/definitions/testcase" } } }, From 10974b5ad90de370fc082db4b74a099d524e82d9 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 19:18:47 +0800 Subject: [PATCH 15/31] fix: json schema validation for testcase v2 --- httprunner/loader/buildup.py | 2 +- httprunner/loader/check.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index 085f3462..65d0042c 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -281,7 +281,6 @@ def load_testsuite(raw_testsuite): } """ - JsonSchemaChecker.validate_testsuite_format(raw_testsuite) raw_testcases = raw_testsuite.pop("testcases") raw_testsuite["testcases"] = {} @@ -293,6 +292,7 @@ def load_testsuite(raw_testsuite): raw_testsuite["testcases"][name] = raw_testcase elif isinstance(raw_testcases, list): + JsonSchemaChecker.validate_testsuite_v2_format(raw_testsuite) # format version 2, implemented in 2.2.0 for raw_testcase in raw_testcases: __extend_with_testcase_ref(raw_testcase) diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 853d1d59..0f01a4fc 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -39,16 +39,6 @@ class JsonSchemaChecker(object): return True - @staticmethod - def validate_testsuite_format(content): - try: - jsonschema.validate(content, testsuite_schema_v2, resolver=resolver) - except jsonschema.exceptions.ValidationError as ex: - logger.log_error(str(ex)) - raise exceptions.FileFormatError - - return True - @staticmethod def validate_testcase_v1_format(content): """ check testcase format v1 if valid @@ -67,6 +57,18 @@ class JsonSchemaChecker(object): return True + @staticmethod + def validate_testsuite_v2_format(content): + """ check testsuite format v2 if valid + """ + try: + jsonschema.validate(content, testsuite_schema_v2, resolver=resolver) + except jsonschema.exceptions.ValidationError as ex: + logger.log_error(str(ex)) + raise exceptions.FileFormatError + + return True + def is_testcase(data_structure): """ check if data_structure is a testcase. From 602a2513172e2e95c5425f74ef3ab19698653686 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 31 Dec 2019 21:51:05 +0800 Subject: [PATCH 16/31] feat: add comparator in validate schema --- httprunner/loader/schemas/common.schema.json | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/httprunner/loader/schemas/common.schema.json b/httprunner/loader/schemas/common.schema.json index 09863f7c..10b610a8 100644 --- a/httprunner/loader/schemas/common.schema.json +++ b/httprunner/loader/schemas/common.schema.json @@ -191,9 +191,18 @@ "type": "object", "propertyNames": { "enum": [ - "eq", - "lt", - "gt" + "eq", "equals", "==", "is", + "lt", "less_than", + "le", "less_than_or_equals", + "gt", "greater_than", + "ge", "greater_than_or_equals", + "ne", "not_equals", + "str_eq", "string_equals", + "len_eq", "length_equals", "count_eq", + "len_gt", "count_gt", "length_greater_than", "count_greater_than", + "len_ge", "count_ge", "length_greater_than_or_equals", "count_greater_than_or_equals", + "len_lt", "count_lt", "length_less_than", "count_less_than", + "len_le", "count_le", "length_less_than_or_equals", "count_less_than_or_equals" ] }, "patternProperties": { From 4995c32e73ba58d22988b83fcf1a96ae4e2154b6 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 09:35:50 +0800 Subject: [PATCH 17/31] fix: json schema validation for testcase v2 --- httprunner/loader/buildup.py | 10 +- httprunner/loader/check.py | 6 ++ httprunner/loader/schemas/common.schema.json | 31 ++++-- .../loader/schemas/testcase.schema.v2.json | 101 ++++++++++++------ tests/test_loader/test_cases.py | 5 +- 5 files changed, 111 insertions(+), 42 deletions(-) diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index 65d0042c..490d3208 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -281,19 +281,21 @@ def load_testsuite(raw_testsuite): } """ - raw_testcases = raw_testsuite.pop("testcases") - raw_testsuite["testcases"] = {} + raw_testcases = raw_testsuite["testcases"] if isinstance(raw_testcases, dict): - # make compatible with version < 2.2.0 + # format version 1, make compatible with version < 2.2.0 + JsonSchemaChecker.validate_testsuite_v1_format(raw_testsuite) + raw_testsuite["testcases"] = {} for name, raw_testcase in raw_testcases.items(): __extend_with_testcase_ref(raw_testcase) raw_testcase.setdefault("name", name) raw_testsuite["testcases"][name] = raw_testcase elif isinstance(raw_testcases, list): - JsonSchemaChecker.validate_testsuite_v2_format(raw_testsuite) # format version 2, implemented in 2.2.0 + JsonSchemaChecker.validate_testsuite_v2_format(raw_testsuite) + raw_testsuite["testcases"] = {} for raw_testcase in raw_testcases: __extend_with_testcase_ref(raw_testcase) testcase_name = raw_testcase["name"] diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 0f01a4fc..44a0f721 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -57,6 +57,12 @@ class JsonSchemaChecker(object): return True + @staticmethod + def validate_testsuite_v1_format(content): + """ check testsuite format v1 if valid + """ + return True + @staticmethod def validate_testsuite_v2_format(content): """ check testsuite format v2 if valid diff --git a/httprunner/loader/schemas/common.schema.json b/httprunner/loader/schemas/common.schema.json index 10b610a8..de03d1cf 100644 --- a/httprunner/loader/schemas/common.schema.json +++ b/httprunner/loader/schemas/common.schema.json @@ -175,13 +175,32 @@ }, "extract": { "description": "used to extract session variables for later requests", - "type": "object", - "patternProperties": { - "^[A-Za-z_][A-Za-z0-9_]*$": { - "description": "extraction rule for session variable, maybe in jsonpath/regex/jmespath", - "type": "string" + "oneOf": [ + { + "type": "object", + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_]*$": { + "description": "extraction rule for session variable, maybe in jsonpath/regex/jmespath", + "type": "string" + } + }, + "minProperties": 1 + }, + { + "type": "array", + "items": { + "type": "object", + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_]*$": { + "description": "extraction rule for session variable, maybe in jsonpath/regex/jmespath", + "type": "string" + } + }, + "minProperties": 1, + "maxProperties": 1 + } } - } + ] }, "validate": { "description": "used to validate response fields", diff --git a/httprunner/loader/schemas/testcase.schema.v2.json b/httprunner/loader/schemas/testcase.schema.v2.json index 4a1f2395..75009f3d 100644 --- a/httprunner/loader/schemas/testcase.schema.v2.json +++ b/httprunner/loader/schemas/testcase.schema.v2.json @@ -6,42 +6,13 @@ "teststep": { "type": "object", "oneOf": [ - { - "properties": { - "name": { - "$ref": "common.schema.json#/definitions/name" - }, - "api": { - "description": "api reference, value is api file relative path", - "type": "string" - }, - "variables": { - "$ref": "common.schema.json#/definitions/variables" - }, - "extract": { - "$ref": "common.schema.json#/definitions/extract" - }, - "validate": { - "$ref": "common.schema.json#/definitions/validate" - }, - "setup_hooks": { - "$ref": "common.schema.json#/definitions/hook" - }, - "teardown_hooks": { - "$ref": "common.schema.json#/definitions/hook" - } - }, - "required": [ - "name", - "api" - ] - }, { "properties": { "name": { "$ref": "common.schema.json#/definitions/name" }, "request": { + "description": "define api request directly", "$ref": "common.schema.json#/definitions/request" }, "variables": { @@ -64,6 +35,76 @@ "name", "request" ] + }, + { + "properties": { + "name": { + "$ref": "common.schema.json#/definitions/name" + }, + "api": { + "description": "api reference, value is api file relative path", + "type": "string" + }, + "variables": { + "$ref": "common.schema.json#/definitions/variables" + }, + "extract": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "$ref": "common.schema.json#/definitions/extract" + } + ] + }, + "validate": { + "$ref": "common.schema.json#/definitions/validate" + }, + "setup_hooks": { + "$ref": "common.schema.json#/definitions/hook" + }, + "teardown_hooks": { + "$ref": "common.schema.json#/definitions/hook" + } + }, + "required": [ + "name", + "api" + ] + }, + { + "properties": { + "name": { + "$ref": "common.schema.json#/definitions/name" + }, + "testcase": { + "description": "testcase reference, value is testcase file relative path", + "type": "string" + }, + "variables": { + "$ref": "common.schema.json#/definitions/variables" + }, + "extract": { + "type": "array", + "items": { + "type": "string" + } + }, + "setup_hooks": { + "$ref": "common.schema.json#/definitions/hook" + }, + "teardown_hooks": { + "$ref": "common.schema.json#/definitions/hook" + } + }, + "required": [ + "name", + "testcase" + ] } ] } diff --git a/tests/test_loader/test_cases.py b/tests/test_loader/test_cases.py index f501a3d0..d2266236 100644 --- a/tests/test_loader/test_cases.py +++ b/tests/test_loader/test_cases.py @@ -279,6 +279,7 @@ class TestSuiteLoader(unittest.TestCase): def test_load_project_tests(self): buildup.load_project_data(os.path.join(os.getcwd(), "tests")) - api_file_path = os.path.join(os.getcwd(), "tests", "api", "get_token.yml") - self.assertIn(api_file_path, self.tests_def_mapping["api"]) + self.assertIn("gen_md5", self.project_mapping["functions"]) self.assertEqual(self.project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH") + self.assertEqual(self.project_mapping["PWD"], os.path.dirname(os.path.dirname(__file__))) + self.assertEqual(self.project_mapping["test_path"], os.path.dirname(os.path.dirname(__file__))) From 8f826e3c02c82a1029864c059f938049e86601cf Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 09:42:47 +0800 Subject: [PATCH 18/31] fix: update comparator schema --- httprunner/loader/schemas/common.schema.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/httprunner/loader/schemas/common.schema.json b/httprunner/loader/schemas/common.schema.json index de03d1cf..9d132b51 100644 --- a/httprunner/loader/schemas/common.schema.json +++ b/httprunner/loader/schemas/common.schema.json @@ -221,7 +221,13 @@ "len_gt", "count_gt", "length_greater_than", "count_greater_than", "len_ge", "count_ge", "length_greater_than_or_equals", "count_greater_than_or_equals", "len_lt", "count_lt", "length_less_than", "count_less_than", - "len_le", "count_le", "length_less_than_or_equals", "count_less_than_or_equals" + "len_le", "count_le", "length_less_than_or_equals", "count_less_than_or_equals", + "contains", + "contained_by", + "type_match", + "regex_match", + "startswith", + "endswith" ] }, "patternProperties": { From dc2487d41830e5585cbb440495a1e91bb8d5da66 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 09:51:43 +0800 Subject: [PATCH 19/31] fix: compatibility for Python 2.7 --- tests/test_loader/test_cases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_loader/test_cases.py b/tests/test_loader/test_cases.py index d2266236..5a4dcea1 100644 --- a/tests/test_loader/test_cases.py +++ b/tests/test_loader/test_cases.py @@ -281,5 +281,5 @@ class TestSuiteLoader(unittest.TestCase): buildup.load_project_data(os.path.join(os.getcwd(), "tests")) self.assertIn("gen_md5", self.project_mapping["functions"]) self.assertEqual(self.project_mapping["env"]["PROJECT_KEY"], "ABCDEFGH") - self.assertEqual(self.project_mapping["PWD"], os.path.dirname(os.path.dirname(__file__))) - self.assertEqual(self.project_mapping["test_path"], os.path.dirname(os.path.dirname(__file__))) + self.assertEqual(self.project_mapping["PWD"], os.path.abspath(os.path.dirname(os.path.dirname(__file__)))) + self.assertEqual(self.project_mapping["test_path"], os.path.abspath(os.path.dirname(os.path.dirname(__file__)))) From 841c74bcfa9e0016667d6c13aab9dd423a482665 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 10:37:16 +0800 Subject: [PATCH 20/31] feat: add json schema validation for testcase v1 --- docs/CHANGELOG.md | 2 +- httprunner/loader/buildup.py | 1 + httprunner/loader/check.py | 10 ++ .../loader/schemas/testcase.schema.v1.json | 138 ++++++++++++++++++ 4 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 httprunner/loader/schemas/testcase.schema.v1.json diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a193c99f..8cd3202c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,7 +5,7 @@ **Added** - feat: add json schema validation for api -- feat: add json schema validation for testcase v2 +- feat: add json schema validation for testcase v1 & v2 - feat: add json schema validation for testsuite v2 **Changed** diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index 490d3208..edc8e92b 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -175,6 +175,7 @@ def load_testcase(raw_testcase): } """ + JsonSchemaChecker.validate_testcase_v1_format(raw_testcase) config = {} tests = [] diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 44a0f721..58c1c90b 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -9,6 +9,7 @@ from httprunner import exceptions, logger schemas_root_dir = os.path.join(os.path.dirname(__file__), "schemas") common_schema_path = os.path.join(schemas_root_dir, "common.schema.json") api_schema_path = os.path.join(schemas_root_dir, "api.schema.json") +testcase_schema_v1_path = os.path.join(schemas_root_dir, "testcase.schema.v1.json") testcase_schema_v2_path = os.path.join(schemas_root_dir, "testcase.schema.v2.json") testsuite_schema_v2_path = os.path.join(schemas_root_dir, "testsuite.schema.v2.json") @@ -19,6 +20,9 @@ with open(common_schema_path) as f: common_schema = json.load(f) resolver = jsonschema.RefResolver("file://{}/".format(os.path.abspath(schemas_root_dir)), common_schema) +with open(testcase_schema_v1_path) as f: + testcase_schema_v1 = json.load(f) + with open(testcase_schema_v2_path) as f: testcase_schema_v2 = json.load(f) @@ -43,6 +47,12 @@ class JsonSchemaChecker(object): def validate_testcase_v1_format(content): """ check testcase format v1 if valid """ + try: + jsonschema.validate(content, testcase_schema_v1, resolver=resolver) + except jsonschema.exceptions.ValidationError as ex: + logger.log_error(str(ex)) + raise exceptions.FileFormatError + return True @staticmethod diff --git a/httprunner/loader/schemas/testcase.schema.v1.json b/httprunner/loader/schemas/testcase.schema.v1.json new file mode 100644 index 00000000..823399cc --- /dev/null +++ b/httprunner/loader/schemas/testcase.schema.v1.json @@ -0,0 +1,138 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "description": "httprunner testcase schema v1 definition", + "type": "array", + "definitions": { + "test": { + "type": "object", + "oneOf": [ + { + "properties": { + "name": { + "$ref": "common.schema.json#/definitions/name" + }, + "request": { + "description": "define api request directly", + "$ref": "common.schema.json#/definitions/request" + }, + "variables": { + "$ref": "common.schema.json#/definitions/variables" + }, + "extract": { + "$ref": "common.schema.json#/definitions/extract" + }, + "validate": { + "$ref": "common.schema.json#/definitions/validate" + }, + "setup_hooks": { + "$ref": "common.schema.json#/definitions/hook" + }, + "teardown_hooks": { + "$ref": "common.schema.json#/definitions/hook" + } + }, + "required": [ + "name", + "request" + ] + }, + { + "properties": { + "name": { + "$ref": "common.schema.json#/definitions/name" + }, + "api": { + "description": "api reference, value is api file relative path", + "type": "string" + }, + "variables": { + "$ref": "common.schema.json#/definitions/variables" + }, + "extract": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "$ref": "common.schema.json#/definitions/extract" + } + ] + }, + "validate": { + "$ref": "common.schema.json#/definitions/validate" + }, + "setup_hooks": { + "$ref": "common.schema.json#/definitions/hook" + }, + "teardown_hooks": { + "$ref": "common.schema.json#/definitions/hook" + } + }, + "required": [ + "name", + "api" + ] + }, + { + "properties": { + "name": { + "$ref": "common.schema.json#/definitions/name" + }, + "testcase": { + "description": "testcase reference, value is testcase file relative path", + "type": "string" + }, + "variables": { + "$ref": "common.schema.json#/definitions/variables" + }, + "extract": { + "type": "array", + "items": { + "type": "string" + } + }, + "setup_hooks": { + "$ref": "common.schema.json#/definitions/hook" + }, + "teardown_hooks": { + "$ref": "common.schema.json#/definitions/hook" + } + }, + "required": [ + "name", + "testcase" + ] + } + ] + } + }, + "items": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "config": { + "$ref": "common.schema.json#/definitions/config" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "test": { + "$ref": "testcase.schema.v1.json#/definitions/test" + } + }, + "additionalProperties": false + } + ], + "minProperties": 1, + "maxProperties": 1 + }, + "minItems": 2 +} \ No newline at end of file From 5f66dc069d64041e6a8f57bf21e9468e998e1bc6 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 21:50:00 +0800 Subject: [PATCH 21/31] feat: add json schema validation for testsutie v1 --- docs/CHANGELOG.md | 2 +- httprunner/loader/check.py | 10 +++ httprunner/loader/schemas/common.schema.json | 71 +++++++++++-------- .../loader/schemas/testsuite.schema.v1.json | 66 +++++++++++++++++ 4 files changed, 117 insertions(+), 32 deletions(-) create mode 100644 httprunner/loader/schemas/testsuite.schema.v1.json diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8cd3202c..f2d1a1c9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,7 +6,7 @@ - feat: add json schema validation for api - feat: add json schema validation for testcase v1 & v2 -- feat: add json schema validation for testsuite v2 +- feat: add json schema validation for testsuite v1 & v2 **Changed** diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 58c1c90b..cf9b9ad8 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -11,6 +11,7 @@ common_schema_path = os.path.join(schemas_root_dir, "common.schema.json") api_schema_path = os.path.join(schemas_root_dir, "api.schema.json") testcase_schema_v1_path = os.path.join(schemas_root_dir, "testcase.schema.v1.json") testcase_schema_v2_path = os.path.join(schemas_root_dir, "testcase.schema.v2.json") +testsuite_schema_v1_path = os.path.join(schemas_root_dir, "testsuite.schema.v1.json") testsuite_schema_v2_path = os.path.join(schemas_root_dir, "testsuite.schema.v2.json") with open(api_schema_path) as f: @@ -26,6 +27,9 @@ with open(testcase_schema_v1_path) as f: with open(testcase_schema_v2_path) as f: testcase_schema_v2 = json.load(f) +with open(testsuite_schema_v1_path) as f: + testsuite_schema_v1 = json.load(f) + with open(testsuite_schema_v2_path) as f: testsuite_schema_v2 = json.load(f) @@ -71,6 +75,12 @@ class JsonSchemaChecker(object): def validate_testsuite_v1_format(content): """ check testsuite format v1 if valid """ + try: + jsonschema.validate(content, testsuite_schema_v1, resolver=resolver) + except jsonschema.exceptions.ValidationError as ex: + logger.log_error(str(ex)) + raise exceptions.FileFormatError + return True @staticmethod diff --git a/httprunner/loader/schemas/common.schema.json b/httprunner/loader/schemas/common.schema.json index 9d132b51..b2e09835 100644 --- a/httprunner/loader/schemas/common.schema.json +++ b/httprunner/loader/schemas/common.schema.json @@ -12,7 +12,19 @@ }, "variables": { "description": "define variables for api/teststep/testcase/testsuite", - "type": "object" + "oneOf": [ + { + "type": "object" + }, + { + "type": "array", + "items": { + "type": "object", + "maxProperties": 1, + "minProperties": 1 + } + } + ] }, "config": { "description": "used in testcase/testsuite to configure common fields", @@ -207,37 +219,34 @@ "type": "array", "items": { "description": "one validator definition", - "type": "object", - "propertyNames": { - "enum": [ - "eq", "equals", "==", "is", - "lt", "less_than", - "le", "less_than_or_equals", - "gt", "greater_than", - "ge", "greater_than_or_equals", - "ne", "not_equals", - "str_eq", "string_equals", - "len_eq", "length_equals", "count_eq", - "len_gt", "count_gt", "length_greater_than", "count_greater_than", - "len_ge", "count_ge", "length_greater_than_or_equals", "count_greater_than_or_equals", - "len_lt", "count_lt", "length_less_than", "count_less_than", - "len_le", "count_le", "length_less_than_or_equals", "count_less_than_or_equals", - "contains", - "contained_by", - "type_match", - "regex_match", - "startswith", - "endswith" - ] - }, - "patternProperties": { - "^[A-Za-z_][A-Za-z0-9_]*$": { - "description": "validate_func_name: [check_value, expect_value]", - "type": "array", - "minItems": 2, - "maxItems": 2 + "oneOf": [ + { + "type": "object", + "properties": { + "check": { + "type": "string" + }, + "comparator": { + "type": "string" + }, + "expect": { + "description": "expected value" + } + }, + "required": ["check", "expect"] + }, + { + "type": "object", + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_]*$": { + "description": "validate_func_name: [check_value, expect_value]", + "type": "array", + "minItems": 2, + "maxItems": 2 + } + } } - } + ] } } } diff --git a/httprunner/loader/schemas/testsuite.schema.v1.json b/httprunner/loader/schemas/testsuite.schema.v1.json new file mode 100644 index 00000000..af9fda01 --- /dev/null +++ b/httprunner/loader/schemas/testsuite.schema.v1.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "description": "httprunner testsuite schema v1 definition", + "type": "object", + "definitions": { + "testcase": { + "type": "object", + "properties": { + "name": { + "$ref": "common.schema.json#/definitions/name" + }, + "variables": { + "$ref": "common.schema.json#/definitions/variables" + }, + "parameters": { + "description": "generate cartesian product variables with parameters, each group of variables will be run once", + "type": "object" + }, + "testcase": { + "description": "testcase reference, value is testcase file relative path", + "type": "string" + } + }, + "required": [ + "testcase" + ] + } + }, + "properties": { + "config": { + "$ref": "common.schema.json#/definitions/config" + }, + "testcases": { + "description": "testcase of a testsuite", + "type": "object", + "minProperties": 1, + "patternProperties": { + ".*": { + "description": "testcase definition", + "$ref": "testsuite.schema.v1.json#/definitions/testcase" + } + } + } + }, + "required": [ + "config", + "testcases" + ], + "examples": [ + { + "config": { + "name": "testsuite name" + }, + "testcases": { + "testcase 1": { + "name": "xxx", + "testcase": "/path/to/testcase1" + }, + "testcase 2": { + "name": "xxx", + "testcase": "/path/to/testcase2" + } + } + } + ] +} \ No newline at end of file From 591b1d70b0cd11bcc3e638c2fa9a1fb85133c7c6 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 21:54:58 +0800 Subject: [PATCH 22/31] refactor: implement public method JsonSchemaChecker.validate_format --- httprunner/loader/check.py | 45 ++++++++++++-------------------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index cf9b9ad8..636c667f 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -37,63 +37,46 @@ with open(testsuite_schema_v2_path) as f: class JsonSchemaChecker(object): @staticmethod - def validate_api_format(content): - + def validate_format(content, scheme): + """ check api/testcase/testsuite format if valid + """ try: - jsonschema.validate(content, api_schema, resolver=resolver) + jsonschema.validate(content, scheme, resolver=resolver) except jsonschema.exceptions.ValidationError as ex: logger.log_error(str(ex)) raise exceptions.FileFormatError return True + @staticmethod + def validate_api_format(content): + """ check api format if valid + """ + return JsonSchemaChecker.validate_format(content, api_schema) + @staticmethod def validate_testcase_v1_format(content): """ check testcase format v1 if valid """ - try: - jsonschema.validate(content, testcase_schema_v1, resolver=resolver) - except jsonschema.exceptions.ValidationError as ex: - logger.log_error(str(ex)) - raise exceptions.FileFormatError - - return True + return JsonSchemaChecker.validate_format(content, testcase_schema_v1) @staticmethod def validate_testcase_v2_format(content): """ check testcase format v2 if valid """ - try: - jsonschema.validate(content, testcase_schema_v2, resolver=resolver) - except jsonschema.exceptions.ValidationError as ex: - logger.log_error(str(ex)) - raise exceptions.FileFormatError - - return True + return JsonSchemaChecker.validate_format(content, testcase_schema_v2) @staticmethod def validate_testsuite_v1_format(content): """ check testsuite format v1 if valid """ - try: - jsonschema.validate(content, testsuite_schema_v1, resolver=resolver) - except jsonschema.exceptions.ValidationError as ex: - logger.log_error(str(ex)) - raise exceptions.FileFormatError - - return True + return JsonSchemaChecker.validate_format(content, testsuite_schema_v1) @staticmethod def validate_testsuite_v2_format(content): """ check testsuite format v2 if valid """ - try: - jsonschema.validate(content, testsuite_schema_v2, resolver=resolver) - except jsonschema.exceptions.ValidationError as ex: - logger.log_error(str(ex)) - raise exceptions.FileFormatError - - return True + return JsonSchemaChecker.validate_format(content, testsuite_schema_v2) def is_testcase(data_structure): From 095f9679f02ccd890f28b736906f7fd2723611a8 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 21:56:07 +0800 Subject: [PATCH 23/31] refactor: remove unused code --- httprunner/loader/check.py | 19 ------------------- tests/test_loader/test_check.py | 14 -------------- 2 files changed, 33 deletions(-) diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 636c667f..888d38d3 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -205,22 +205,3 @@ def is_function(item): """ Takes item object, returns True if it is a function. """ return isinstance(item, types.FunctionType) - - -def is_variable(tup): - """ Takes (name, object) tuple, returns True if it is a variable. - """ - name, item = tup - if callable(item): - # function or class - return False - - if isinstance(item, types.ModuleType): - # imported module - return False - - if name.startswith("_"): - # private property - return False - - return True diff --git a/tests/test_loader/test_check.py b/tests/test_loader/test_check.py index 5ba16571..6160ad88 100644 --- a/tests/test_loader/test_check.py +++ b/tests/test_loader/test_check.py @@ -60,17 +60,3 @@ class TestLoaderCheck(unittest.TestCase): } ] self.assertTrue(data_structure) - - def test_is_variable(self): - var1 = 123 - var2 = "abc" - self.assertTrue(check.is_variable(("var1", var1))) - self.assertTrue(check.is_variable(("var2", var2))) - - __var = 123 - self.assertFalse(check.is_variable(("__var", __var))) - - func = lambda x: x + 1 - self.assertFalse(check.is_variable(("func", func))) - - self.assertFalse(check.is_variable(("unittest", unittest))) From f8602e0edf8e53a284bc699e5b0295ecfcab8a61 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 21:59:41 +0800 Subject: [PATCH 24/31] refactor: check if is function --- httprunner/loader/check.py | 7 ------- httprunner/loader/load.py | 4 ++-- tests/test_loader/test_check.py | 5 ----- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 888d38d3..2f7720f4 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -1,6 +1,5 @@ import json import os -import types import jsonschema @@ -199,9 +198,3 @@ def is_testcase_path(path): return False return True - - -def is_function(item): - """ Takes item object, returns True if it is a function. - """ - return isinstance(item, types.FunctionType) diff --git a/httprunner/loader/load.py b/httprunner/loader/load.py index bf75d20e..27fe0455 100644 --- a/httprunner/loader/load.py +++ b/httprunner/loader/load.py @@ -2,12 +2,12 @@ import csv import io import json import os +import types import yaml from httprunner import builtin from httprunner import exceptions, logger, utils -from httprunner.loader.check import is_function from httprunner.loader.locate import get_project_working_directory try: @@ -206,7 +206,7 @@ def load_module_functions(module): module_functions = {} for name, item in vars(module).items(): - if is_function(item): + if isinstance(item, types.FunctionType): module_functions[name] = item return module_functions diff --git a/tests/test_loader/test_check.py b/tests/test_loader/test_check.py index 6160ad88..82325a96 100644 --- a/tests/test_loader/test_check.py +++ b/tests/test_loader/test_check.py @@ -5,11 +5,6 @@ from httprunner.loader import check class TestLoaderCheck(unittest.TestCase): - def test_is_function(self): - func = lambda x: x + 1 - self.assertTrue(check.is_function(func)) - self.assertTrue(check.is_function(check.is_testcase)) - def test_is_testcases(self): data_structure = "path/to/file" self.assertFalse(check.is_testcases(data_structure)) From dbcb3bdb8dde967b482883b1f4e9ec519a90a453 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 22:02:29 +0800 Subject: [PATCH 25/31] refactor: make JsonSchemaChecker as loader module function --- httprunner/loader/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/httprunner/loader/__init__.py b/httprunner/loader/__init__.py index 4b6ecccd..6c529894 100644 --- a/httprunner/loader/__init__.py +++ b/httprunner/loader/__init__.py @@ -8,7 +8,7 @@ HttpRunner loader """ -from httprunner.loader.check import is_testcase_path, is_testcases +from httprunner.loader.check import is_testcase_path, is_testcases, JsonSchemaChecker from httprunner.loader.locate import get_project_working_directory as get_pwd from httprunner.loader.load import load_csv_file, load_builtin_functions from httprunner.loader.buildup import load_cases, load_project_data @@ -16,6 +16,7 @@ from httprunner.loader.buildup import load_cases, load_project_data __all__ = [ "is_testcase_path", "is_testcases", + "JsonSchemaChecker", "get_pwd", "load_csv_file", "load_builtin_functions", From b1bf06d4045087f2c6a9cd5970b74673f9bc9ef0 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 22:13:50 +0800 Subject: [PATCH 26/31] doc: remove complete TODOs --- httprunner/loader/buildup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/httprunner/loader/buildup.py b/httprunner/loader/buildup.py index edc8e92b..23f772cf 100644 --- a/httprunner/loader/buildup.py +++ b/httprunner/loader/buildup.py @@ -349,7 +349,6 @@ def load_test_file(path): if "testcases" in raw_content: # file_type: testsuite - # TODO: add json schema validation for testsuite loaded_content = load_testsuite(raw_content) loaded_content["path"] = path loaded_content["type"] = "testsuite" @@ -374,7 +373,6 @@ def load_test_file(path): elif isinstance(raw_content, list) and len(raw_content) > 0: # file_type: testcase # make compatible with version < 2.2.0 - # TODO: add json schema validation for testcase loaded_content = load_testcase(raw_content) loaded_content["path"] = path loaded_content["type"] = "testcase" From 3ccf68061ecc3e64e6afba9190cfc7ddab142bb5 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 22:17:49 +0800 Subject: [PATCH 27/31] feat: check if testfile path, should be json or yaml file --- httprunner/loader/check.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 2f7720f4..5c0ca459 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -197,4 +197,8 @@ def is_testcase_path(path): if not os.path.exists(path): return False + file_suffix = os.path.splitext(path)[1].lower() + if file_suffix != ['.json', '.yaml', '.yml']: + return False + return True From b5be7f0cb4210d0cd58cf3333bb79a8e7d163671 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 22:34:41 +0800 Subject: [PATCH 28/31] refactor: is_test_path --- httprunner/api.py | 2 +- httprunner/loader/__init__.py | 4 ++-- httprunner/loader/check.py | 35 ++++++++++++++++++++++++----------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index cbb7fc0f..fa4fe0d7 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -289,7 +289,7 @@ class HttpRunner(object): """ logger.log_info("HttpRunner version: {}".format(__version__)) - if loader.is_testcase_path(path_or_tests): + if loader.is_test_path(path_or_tests): return self.run_path(path_or_tests, dot_env_path, mapping) elif loader.is_testcases(path_or_tests): return self.run_tests(path_or_tests) diff --git a/httprunner/loader/__init__.py b/httprunner/loader/__init__.py index 6c529894..c6be3e94 100644 --- a/httprunner/loader/__init__.py +++ b/httprunner/loader/__init__.py @@ -8,13 +8,13 @@ HttpRunner loader """ -from httprunner.loader.check import is_testcase_path, is_testcases, JsonSchemaChecker +from httprunner.loader.check import is_test_path, is_testcases, JsonSchemaChecker from httprunner.loader.locate import get_project_working_directory as get_pwd from httprunner.loader.load import load_csv_file, load_builtin_functions from httprunner.loader.buildup import load_cases, load_project_data __all__ = [ - "is_testcase_path", + "is_test_path", "is_testcases", "JsonSchemaChecker", "get_pwd", diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 5c0ca459..61e4c052 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -175,30 +175,43 @@ def is_testcases(data_structure): return True -def is_testcase_path(path): - """ check if path is testcase path or path list. +def is_test_path(path): + """ check if path is valid json/yaml file path or a existed directory. Args: - path (str/list): file path or file path list. + path (str/list/tuple): file path/directory or file path list. Returns: bool: True if path is valid file path or path list, otherwise False. """ - if not isinstance(path, (str, list)): + if not isinstance(path, (str, list, tuple)): return False - if isinstance(path, list): + elif isinstance(path, (list, tuple)): for p in path: - if not is_testcase_path(p): + if not is_test_path(p): return False - if isinstance(path, str): + return True + + else: + # path is string if not os.path.exists(path): return False - file_suffix = os.path.splitext(path)[1].lower() - if file_suffix != ['.json', '.yaml', '.yml']: + # path exists + if os.path.isfile(path): + # path is a file + file_suffix = os.path.splitext(path)[1].lower() + if file_suffix not in ['.json', '.yaml', '.yml']: + # path is not json/yaml file + return False + else: + return True + elif os.path.isdir(path): + # path is a directory + return True + else: + # path is neither a folder nor a file, maybe a symbol link or something else return False - - return True From 46c3a43187bafb0721574c5c05c85c03515e1ca2 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 23:19:29 +0800 Subject: [PATCH 29/31] refactor: is_test_content --- httprunner/api.py | 2 +- httprunner/loader/__init__.py | 4 +- httprunner/loader/check.py | 183 +++++++++---------- httprunner/loader/schemas/common.schema.json | 3 +- tests/test_loader/test_check.py | 28 +-- 5 files changed, 98 insertions(+), 122 deletions(-) diff --git a/httprunner/api.py b/httprunner/api.py index fa4fe0d7..11666fed 100644 --- a/httprunner/api.py +++ b/httprunner/api.py @@ -291,7 +291,7 @@ class HttpRunner(object): logger.log_info("HttpRunner version: {}".format(__version__)) if loader.is_test_path(path_or_tests): return self.run_path(path_or_tests, dot_env_path, mapping) - elif loader.is_testcases(path_or_tests): + elif loader.is_test_content(path_or_tests): return self.run_tests(path_or_tests) else: raise exceptions.ParamsError("Invalid testcase path or testcases: {}".format(path_or_tests)) diff --git a/httprunner/loader/__init__.py b/httprunner/loader/__init__.py index c6be3e94..36f44798 100644 --- a/httprunner/loader/__init__.py +++ b/httprunner/loader/__init__.py @@ -8,14 +8,14 @@ HttpRunner loader """ -from httprunner.loader.check import is_test_path, is_testcases, JsonSchemaChecker +from httprunner.loader.check import is_test_path, is_test_content, JsonSchemaChecker from httprunner.loader.locate import get_project_working_directory as get_pwd from httprunner.loader.load import load_csv_file, load_builtin_functions from httprunner.loader.buildup import load_cases, load_project_data __all__ = [ "is_test_path", - "is_testcases", + "is_test_content", "JsonSchemaChecker", "get_pwd", "load_csv_file", diff --git a/httprunner/loader/check.py b/httprunner/loader/check.py index 61e4c052..b011be1a 100644 --- a/httprunner/loader/check.py +++ b/httprunner/loader/check.py @@ -78,103 +78,6 @@ class JsonSchemaChecker(object): return JsonSchemaChecker.validate_format(content, testsuite_schema_v2) -def is_testcase(data_structure): - """ check if data_structure is a testcase. - - Args: - data_structure (dict): testcase should always be in the following data structure: - - { - "config": { - "name": "desc1", - "variables": [], # optional - "request": {} # optional - }, - "teststeps": [ - test_dict1, - { # test_dict2 - 'name': 'test step desc2', - 'variables': [], # optional - 'extract': [], # optional - 'validate': [], - 'request': {}, - 'function_meta': {} - } - ] - } - - Returns: - bool: True if data_structure is valid testcase, otherwise False. - - """ - # TODO: replace with JSON schema validation - if not isinstance(data_structure, dict): - return False - - if "teststeps" not in data_structure: - return False - - if not isinstance(data_structure["teststeps"], list): - return False - - return True - - -def is_testcases(data_structure): - """ check if data_structure is testcase or testcases list. - - Args: - data_structure (dict): testcase(s) should always be in the following data structure: - { - "project_mapping": { - "PWD": "XXXXX", - "functions": {}, - "env": {} - }, - "testcases": [ - { # testcase data structure - "config": { - "name": "desc1", - "path": "testcase1_path", - "variables": [], # optional - }, - "teststeps": [ - # test data structure - { - 'name': 'test step desc1', - 'variables': [], # optional - 'extract': [], # optional - 'validate': [], - 'request': {} - }, - test_dict_2 # another test dict - ] - }, - testcase_dict_2 # another testcase dict - ] - } - - Returns: - bool: True if data_structure is valid testcase(s), otherwise False. - - """ - if not isinstance(data_structure, dict): - return False - - if "testcases" not in data_structure: - return False - - testcases = data_structure["testcases"] - if not isinstance(testcases, list): - return False - - for item in testcases: - if not is_testcase(item): - return False - - return True - - def is_test_path(path): """ check if path is valid json/yaml file path or a existed directory. @@ -215,3 +118,89 @@ def is_test_path(path): else: # path is neither a folder nor a file, maybe a symbol link or something else return False + + +def is_test_content(data_structure): + """ check if data_structure is apis/testcases/testsuites. + + Args: + data_structure (dict): should include keys, apis or testcases or testsuites + + Returns: + bool: True if data_structure is valid apis/testcases/testsuites, otherwise False. + + """ + if not isinstance(data_structure, dict): + return False + + if "apis" in data_structure: + # maybe a group of api content + apis = data_structure["apis"] + if not isinstance(apis, list): + return False + + for item in apis: + is_testcase = False + try: + JsonSchemaChecker.validate_api_format(item) + is_testcase = True + except exceptions.FileFormatError: + pass + + if not is_testcase: + return False + + return True + + elif "testcases" in data_structure: + # maybe a testsuite, containing a group of testcases + testcases = data_structure["testcases"] + if not isinstance(testcases, list): + return False + + for item in testcases: + is_testcase = False + try: + JsonSchemaChecker.validate_testcase_v2_format(item) + is_testcase = True + except exceptions.FileFormatError: + pass + + try: + JsonSchemaChecker.validate_testcase_v2_format(item) + is_testcase = True + except exceptions.FileFormatError: + pass + + if not is_testcase: + return False + + return True + + elif "testsuites" in data_structure: + # maybe a group of testsuites + testsuites = data_structure["testsuites"] + if not isinstance(testsuites, list): + return False + + for item in testsuites: + is_testcase = False + try: + JsonSchemaChecker.validate_testsuite_v1_format(item) + is_testcase = True + except exceptions.FileFormatError: + pass + + try: + JsonSchemaChecker.validate_testsuite_v2_format(item) + is_testcase = True + except exceptions.FileFormatError: + pass + + if not is_testcase: + return False + + return True + + else: + return False diff --git a/httprunner/loader/schemas/common.schema.json b/httprunner/loader/schemas/common.schema.json index b2e09835..0cbf6038 100644 --- a/httprunner/loader/schemas/common.schema.json +++ b/httprunner/loader/schemas/common.schema.json @@ -195,8 +195,7 @@ "description": "extraction rule for session variable, maybe in jsonpath/regex/jmespath", "type": "string" } - }, - "minProperties": 1 + } }, { "type": "array", diff --git a/tests/test_loader/test_check.py b/tests/test_loader/test_check.py index 82325a96..62de19b1 100644 --- a/tests/test_loader/test_check.py +++ b/tests/test_loader/test_check.py @@ -7,9 +7,9 @@ class TestLoaderCheck(unittest.TestCase): def test_is_testcases(self): data_structure = "path/to/file" - self.assertFalse(check.is_testcases(data_structure)) + self.assertFalse(check.is_test_content(data_structure)) data_structure = ["path/to/file1", "path/to/file2"] - self.assertFalse(check.is_testcases(data_structure)) + self.assertFalse(check.is_test_content(data_structure)) data_structure = { "project_mapping": { @@ -29,9 +29,12 @@ class TestLoaderCheck(unittest.TestCase): { 'name': 'test step desc1', 'variables': [], # optional - 'extract': [], # optional + 'extract': {}, # optional 'validate': [], - 'request': {} + 'request': { + "method": "GET", + "url": "https://docs.httprunner.org" + } }, # test_dict2 # another test dict ] @@ -39,19 +42,4 @@ class TestLoaderCheck(unittest.TestCase): # testcase_dict_2 # another testcase dict ] } - self.assertTrue(check.is_testcases(data_structure)) - data_structure = [ - { - "name": "desc1", - "config": {}, - "api": {}, - "testcases": ["testcase11", "testcase12"] - }, - { - "name": "desc2", - "config": {}, - "api": {}, - "testcases": ["testcase21", "testcase22"] - } - ] - self.assertTrue(data_structure) + self.assertTrue(check.is_test_content(data_structure)) From a3f31e8e60d0062b42fbe35b1692c4090f7539c1 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 23:25:03 +0800 Subject: [PATCH 30/31] doc: update changelog --- docs/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f2d1a1c9..0384b5d4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -11,6 +11,8 @@ **Changed** - refactor: use loader.load_cases to validate test files +- refactor: use is_test_path to check if path is valid json/yaml file or a existed directory +- refactor: use is_test_content to check if data_structure is apis/testcases/testsuites ## 2.4.9 (2019-12-29) From d5328e0da609c20589752f47d3c3047aed4a741e Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 1 Jan 2020 23:25:36 +0800 Subject: [PATCH 31/31] doc: update changelog --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0384b5d4..e2a1294b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 2.5.0 (2019-12-31) +## 2.5.0 (2020-01-01) **Added**