diff --git a/httprunner/loader.py b/httprunner/loader.py new file mode 100644 index 00000000..03fc8ae1 --- /dev/null +++ b/httprunner/loader.py @@ -0,0 +1,136 @@ +import csv +import io +import json +import os + +import yaml +from httprunner import exceptions, logger + + +############################################################################### +## file loader +############################################################################### + + +def _check_format(file_path, content): + """ check testcase format if valid + """ + # TODO: replace with JSON schema validation + if not content: + # testcase file content is empty + err_msg = u"Testcase file content is empty: {}".format(file_path) + logger.log_error(err_msg) + raise exceptions.FileFormatError(err_msg) + + elif not isinstance(content, (list, dict)): + # testcase file content does not match testcase format + err_msg = u"Testcase file content format invalid: {}".format(file_path) + logger.log_error(err_msg) + raise exceptions.FileFormatError(err_msg) + + +def load_yaml_file(yaml_file): + """ load yaml file and check file content format + """ + with io.open(yaml_file, 'r', encoding='utf-8') as stream: + yaml_content = yaml.load(stream) + _check_format(yaml_file, yaml_content) + return yaml_content + + +def load_json_file(json_file): + """ load json file and check file content format + """ + with io.open(json_file, encoding='utf-8') as data_file: + try: + json_content = json.load(data_file) + except exceptions.JSONDecodeError: + err_msg = u"JSONDecodeError: JSON file format error: {}".format(json_file) + logger.log_error(err_msg) + raise exceptions.FileFormatError(err_msg) + + _check_format(json_file, json_content) + return json_content + + +def load_csv_file(csv_file): + """ load csv file and check file content format + @param + csv_file: csv file path + e.g. csv file content: + username,password + test1,111111 + test2,222222 + test3,333333 + @return + list of parameter, each parameter is in dict format + e.g. + [ + {'username': 'test1', 'password': '111111'}, + {'username': 'test2', 'password': '222222'}, + {'username': 'test3', 'password': '333333'} + ] + """ + csv_content_list = [] + + with io.open(csv_file, encoding='utf-8') as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + csv_content_list.append(row) + + return csv_content_list + + +def load_file(file_path): + if not os.path.isfile(file_path): + raise exceptions.FileNotFound("{} does not exist.".format(file_path)) + + file_suffix = os.path.splitext(file_path)[1].lower() + if file_suffix == '.json': + return load_json_file(file_path) + elif file_suffix in ['.yaml', '.yml']: + return load_yaml_file(file_path) + elif file_suffix == ".csv": + return load_csv_file(file_path) + else: + # '' or other suffix + err_msg = u"Unsupported file format: {}".format(file_path) + logger.log_warning(err_msg) + return [] + + +def load_folder_files(folder_path, recursive=True): + """ load folder path, return all files in list format. + @param + folder_path: specified folder path to load + recursive: if True, will load files recursively + """ + if isinstance(folder_path, (list, set)): + files = [] + for path in set(folder_path): + files.extend(load_folder_files(path, recursive)) + + return files + + if not os.path.exists(folder_path): + return [] + + file_list = [] + + for dirpath, dirnames, filenames in os.walk(folder_path): + filenames_list = [] + + for filename in filenames: + if not filename.endswith(('.yml', '.yaml', '.json')): + continue + + filenames_list.append(filename) + + for filename in filenames_list: + file_path = os.path.join(dirpath, filename) + file_list.append(file_path) + + if not recursive: + break + + return file_list diff --git a/httprunner/testcase.py b/httprunner/testcase.py index b7a52a2e..140b7355 100644 --- a/httprunner/testcase.py +++ b/httprunner/testcase.py @@ -9,10 +9,9 @@ import os import random import re -from httprunner import exceptions, logger, utils +from httprunner import exceptions, loader, logger, utils from httprunner.compat import (OrderedDict, basestring, builtin_str, numeric_types, str) -from httprunner.utils import FileUtils variable_regexp = r"\$([\w_]+)" function_regexp = r"\$\{([\w_]+\([\$\w\.\-_ =,]*\))\}" @@ -119,12 +118,12 @@ class TestcaseLoader(object): # TODO: cache api and suite loading # load api definitions api_def_folder = os.path.join(os.getcwd(), "tests", "api") - for test_file in FileUtils.load_folder_files(api_def_folder): + for test_file in loader.load_folder_files(api_def_folder): TestcaseLoader.load_api_file(test_file) # load suite definitions suite_def_folder = os.path.join(os.getcwd(), "tests", "suite") - for suite_file in FileUtils.load_folder_files(suite_def_folder): + for suite_file in loader.load_folder_files(suite_def_folder): suite = TestcaseLoader.load_test_file(suite_file) if "def" not in suite["config"]: raise exceptions.ParamsError("def missed in suite file: {}!".format(suite_file)) @@ -155,7 +154,7 @@ class TestcaseLoader(object): } ] """ - api_items = FileUtils.load_file(file_path) + api_items = loader.load_file(file_path) if not isinstance(api_items, list): raise exceptions.FileFormatError("API format error: {}".format(file_path)) @@ -217,7 +216,7 @@ class TestcaseLoader(object): }, "testcases": [] # TODO: rename to tests } - for item in FileUtils.load_file(file_path): + for item in loader.load_file(file_path): if not isinstance(item, dict) or len(item) != 1: raise exceptions.FileFormatError("Testcase format error: {}".format(file_path)) @@ -371,7 +370,7 @@ class TestcaseLoader(object): return TestcaseLoader.testcases_cache_mapping[path] if os.path.isdir(path): - files_list = FileUtils.load_folder_files(path) + files_list = loader.load_folder_files(path) testcases_list = TestcaseLoader.load_testsets_by_path(files_list) elif os.path.isfile(path): @@ -792,7 +791,7 @@ class TestcaseParser(object): os.path.dirname(self.file_path), "{}".format(csv_file_name) ) - csv_content_list = FileUtils.load_file(parameter_file_path) + csv_content_list = loader.load_file(parameter_file_path) if fetch_method.lower() == "random": random.shuffle(csv_content_list) diff --git a/httprunner/utils.py b/httprunner/utils.py index edc98c73..eefc2477 100644 --- a/httprunner/utils.py +++ b/httprunner/utils.py @@ -1,7 +1,6 @@ # encoding: utf-8 import copy -import csv import hashlib import hmac import imp @@ -10,12 +9,10 @@ import io import json import os.path import random -import re import string import types from datetime import datetime -import yaml from httprunner import exceptions, logger from httprunner.compat import OrderedDict, basestring, is_py2, is_py3, str from requests.structures import CaseInsensitiveDict @@ -44,132 +41,6 @@ def remove_prefix(text, prefix): return text -class FileUtils(object): - - @staticmethod - def _check_format(file_path, content): - """ check testcase format if valid - """ - if not content: - # testcase file content is empty - err_msg = u"Testcase file content is empty: {}".format(file_path) - logger.log_error(err_msg) - raise exceptions.FileFormatError(err_msg) - - elif not isinstance(content, (list, dict)): - # testcase file content does not match testcase format - err_msg = u"Testcase file content format invalid: {}".format(file_path) - logger.log_error(err_msg) - raise exceptions.FileFormatError(err_msg) - - @staticmethod - def _load_yaml_file(yaml_file): - """ load yaml file and check file content format - """ - with io.open(yaml_file, 'r', encoding='utf-8') as stream: - yaml_content = yaml.load(stream) - FileUtils._check_format(yaml_file, yaml_content) - return yaml_content - - @staticmethod - def _load_json_file(json_file): - """ load json file and check file content format - """ - with io.open(json_file, encoding='utf-8') as data_file: - try: - json_content = json.load(data_file) - except exceptions.JSONDecodeError: - err_msg = u"JSONDecodeError: JSON file format error: {}".format(json_file) - logger.log_error(err_msg) - raise exceptions.FileFormatError(err_msg) - - FileUtils._check_format(json_file, json_content) - return json_content - - @staticmethod - def _load_csv_file(csv_file): - """ load csv file and check file content format - @param - csv_file: csv file path - e.g. csv file content: - username,password - test1,111111 - test2,222222 - test3,333333 - @return - list of parameter, each parameter is in dict format - e.g. - [ - {'username': 'test1', 'password': '111111'}, - {'username': 'test2', 'password': '222222'}, - {'username': 'test3', 'password': '333333'} - ] - """ - csv_content_list = [] - - with io.open(csv_file, encoding='utf-8') as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - csv_content_list.append(row) - - return csv_content_list - - @staticmethod - def load_file(file_path): - if not os.path.isfile(file_path): - raise exceptions.FileNotFound("{} does not exist.".format(file_path)) - - file_suffix = os.path.splitext(file_path)[1].lower() - if file_suffix == '.json': - return FileUtils._load_json_file(file_path) - elif file_suffix in ['.yaml', '.yml']: - return FileUtils._load_yaml_file(file_path) - elif file_suffix == ".csv": - return FileUtils._load_csv_file(file_path) - else: - # '' or other suffix - err_msg = u"Unsupported file format: {}".format(file_path) - logger.log_warning(err_msg) - return [] - - @staticmethod - def load_folder_files(folder_path, recursive=True): - """ load folder path, return all files in list format. - @param - folder_path: specified folder path to load - recursive: if True, will load files recursively - """ - if isinstance(folder_path, (list, set)): - files = [] - for path in set(folder_path): - files.extend(FileUtils.load_folder_files(path, recursive)) - - return files - - if not os.path.exists(folder_path): - return [] - - file_list = [] - - for dirpath, dirnames, filenames in os.walk(folder_path): - filenames_list = [] - - for filename in filenames: - if not filename.endswith(('.yml', '.yaml', '.json')): - continue - - filenames_list.append(filename) - - for filename in filenames_list: - file_path = os.path.join(dirpath, filename) - file_list.append(file_path) - - if not recursive: - break - - return file_list - - def query_json(json_content, query, delimiter='.'): """ Do an xpath-like query with json_content. @param (dict/list/string) json_content diff --git a/tests/test_context.py b/tests/test_context.py index 7e47d596..0e012f24 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -2,9 +2,9 @@ import os import time import requests -from httprunner import exceptions, response, runner, testcase +from httprunner import exceptions, loader, response, runner, testcase from httprunner.context import Context -from httprunner.utils import FileUtils, gen_md5 +from httprunner.utils import gen_md5 from tests.base import ApiServerUnittest @@ -13,7 +13,7 @@ class VariableBindsUnittest(ApiServerUnittest): def setUp(self): self.context = Context() testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml') - self.testcases = FileUtils.load_file(testcase_file_path) + self.testcases = loader.load_file(testcase_file_path) def test_context_init_functions(self): self.assertIn("get_timestamp", self.context.testset_functions_config) diff --git a/tests/test_runner.py b/tests/test_runner.py index a1ad5194..95552512 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,9 +1,9 @@ import os import time -from httprunner import HttpRunner, exceptions, runner +from httprunner import HttpRunner, exceptions, loader, runner from httprunner.testcase import TestcaseLoader -from httprunner.utils import FileUtils, deep_update_dict +from httprunner.utils import deep_update_dict from tests.base import ApiServerUnittest @@ -27,7 +27,7 @@ class TestRunner(ApiServerUnittest): def test_run_single_testcase(self): for testcase_file_path in self.testcase_file_path_list: - testcases = FileUtils.load_file(testcase_file_path) + testcases = loader.load_file(testcase_file_path) config_dict = { "path": testcase_file_path @@ -393,7 +393,7 @@ class TestRunner(ApiServerUnittest): def test_bugfix_type_match(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/test_bugfix.yml') - testcases = FileUtils.load_file(testcase_file_path) + testcases = loader.load_file(testcase_file_path) config_dict = { "path": testcase_file_path } diff --git a/tests/test_utils.py b/tests/test_utils.py index 7b6e24dc..e2febcb7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,138 +4,9 @@ import unittest from httprunner import exceptions, utils from httprunner.compat import OrderedDict -from httprunner.utils import FileUtils from tests.base import ApiServerUnittest -class TestFileUtils(unittest.TestCase): - - def test_load_yaml_file_file_format_error(self): - yaml_tmp_file = "tests/data/tmp.yml" - # create empty yaml file - with open(yaml_tmp_file, 'w') as f: - f.write("") - - with self.assertRaises(exceptions.FileFormatError): - FileUtils._load_yaml_file(yaml_tmp_file) - - os.remove(yaml_tmp_file) - - # create invalid format yaml file - with open(yaml_tmp_file, 'w') as f: - f.write("abc") - - with self.assertRaises(exceptions.FileFormatError): - FileUtils._load_yaml_file(yaml_tmp_file) - - os.remove(yaml_tmp_file) - - - def test_load_json_file_file_format_error(self): - json_tmp_file = "tests/data/tmp.json" - # create empty file - with open(json_tmp_file, 'w') as f: - f.write("") - - with self.assertRaises(exceptions.FileFormatError): - FileUtils._load_json_file(json_tmp_file) - - os.remove(json_tmp_file) - - # create empty json file - with open(json_tmp_file, 'w') as f: - f.write("{}") - - with self.assertRaises(exceptions.FileFormatError): - FileUtils._load_json_file(json_tmp_file) - - os.remove(json_tmp_file) - - # create invalid format json file - with open(json_tmp_file, 'w') as f: - f.write("abc") - - with self.assertRaises(exceptions.FileFormatError): - FileUtils._load_json_file(json_tmp_file) - - os.remove(json_tmp_file) - - def test_load_testcases_bad_filepath(self): - testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo') - with self.assertRaises(exceptions.FileNotFound): - FileUtils.load_file(testcase_file_path) - - def test_load_json_testcases(self): - testcase_file_path = os.path.join( - os.getcwd(), 'tests/data/demo_testset_hardcode.json') - testcases = FileUtils.load_file(testcase_file_path) - self.assertEqual(len(testcases), 3) - test = testcases[0]["test"] - self.assertIn('name', test) - self.assertIn('request', test) - self.assertIn('url', test['request']) - self.assertIn('method', test['request']) - - def test_load_yaml_testcases(self): - testcase_file_path = os.path.join( - os.getcwd(), 'tests/data/demo_testset_hardcode.yml') - testcases = FileUtils.load_file(testcase_file_path) - self.assertEqual(len(testcases), 3) - test = testcases[0]["test"] - self.assertIn('name', test) - self.assertIn('request', test) - self.assertIn('url', test['request']) - self.assertIn('method', test['request']) - - def test_load_csv_file_one_parameter(self): - csv_file_path = os.path.join( - os.getcwd(), 'tests/data/user_agent.csv') - csv_content = FileUtils.load_file(csv_file_path) - self.assertEqual( - csv_content, - [ - {'user_agent': 'iOS/10.1'}, - {'user_agent': 'iOS/10.2'}, - {'user_agent': 'iOS/10.3'} - ] - ) - - def test_load_csv_file_multiple_parameters(self): - csv_file_path = os.path.join( - os.getcwd(), 'tests/data/account.csv') - csv_content = FileUtils.load_file(csv_file_path) - self.assertEqual( - csv_content, - [ - {'username': 'test1', 'password': '111111'}, - {'username': 'test2', 'password': '222222'}, - {'username': 'test3', 'password': '333333'} - ] - ) - - def test_load_folder_files(self): - folder = os.path.join(os.getcwd(), 'tests') - file1 = os.path.join(os.getcwd(), 'tests', 'test_utils.py') - file2 = os.path.join(os.getcwd(), 'tests', 'data', 'demo_binds.yml') - - files = FileUtils.load_folder_files(folder, recursive=False) - self.assertNotIn(file2, files) - - files = FileUtils.load_folder_files(folder) - self.assertIn(file2, files) - self.assertNotIn(file1, files) - - files = FileUtils.load_folder_files(folder) - api_file = os.path.join(os.getcwd(), 'tests', 'api', 'basic.yml') - self.assertIn(api_file, files) - - files = FileUtils.load_folder_files("not_existed_foulder", recursive=False) - self.assertEqual([], files) - - files = FileUtils.load_folder_files(file2, recursive=False) - self.assertEqual([], files) - - class TestUtils(ApiServerUnittest): def test_remove_prefix(self):