new feature: support parameters and data driven

httprunner/__init__.py

@@ -1 +1 @@
-__version__ = '0.9.1'
+__version__ = '0.9.2'

httprunner/task.py

@@ -49,13 +49,26 @@ class ApiTestSuite(unittest.TestSuite):
         super(ApiTestSuite, self).__init__()
 
         self.config_dict = testset.get("config", {})
+
         variables = self.config_dict.get("variables", [])
         variables_mapping = variables_mapping or {}
         self.config_dict["variables"] = utils.override_variables_binds(variables, variables_mapping)
 
-        self.test_runner = runner.Runner(self.config_dict, http_client_session)
+        parameters = self.config_dict.get("parameters", [])
-        testcases = testset.get("testcases", [])
+        cartesian_product_parameters = testcase.gen_cartesian_product_parameters(
-        self._add_tests_to_suite(testcases)
+            parameters,
+            self.config_dict["path"]
+        ) or [{}]
+        for parameter_mapping in cartesian_product_parameters:
+            if parameter_mapping:
+                self.config_dict["variables"] = utils.override_variables_binds(
+                    self.config_dict["variables"],
+                    parameter_mapping
+                )
+
+            self.test_runner = runner.Runner(self.config_dict, http_client_session)
+            testcases = testset.get("testcases", [])
+            self._add_tests_to_suite(testcases)
 
     def _add_tests_to_suite(self, testcases):
         for testcase_dict in testcases:

httprunner/testcase.py

@@ -1,8 +1,10 @@
 import ast
 import io
+import itertools
 import json
 import logging
 import os
+import random
 import re
 from collections import OrderedDict
 
@@ -42,12 +44,70 @@ def _load_json_file(json_file):
         check_format(json_file, json_content)
         return json_content
 
+def _load_csv_file(csv_file):
+    """ load csv file and check file content format
+    @param
+        csv_file: csv file path
+        e.g. csv file content:
+            username,password
+            test1,111111
+            test2,222222
+            test3,333333
+    @return
+        list of parameter, each parameter is in dict format
+        e.g.
+        [
+            {'username': 'test1', 'password': '111111'},
+            {'username': 'test2', 'password': '222222'},
+            {'username': 'test3', 'password': '333333'}
+        ]
+    """
+    csv_content_list = []
+    parameter_list = None
+    collums_num = 0
+    with io.open(csv_file, encoding='utf-8') as data_file:
+        for line in data_file:
+            line_data = line.strip().split(",")
+            if line_data == [""]:
+                # ignore empty line
+                continue
+
+            if not parameter_list:
+                # first line will always be parameter name
+                expected_filename = "{}.csv".format("-".join(line_data))
+                if not csv_file.endswith(expected_filename):
+                    raise exception.FileFormatError("CSV file name does not match with headers: {}".format(csv_file))
+
+                parameter_list = line_data
+                collums_num = len(parameter_list)
+                continue
+
+            # from the second line
+            if len(line_data) != collums_num:
+                err_msg = "CSV file collums does match with headers.\n"
+                err_msg += "\tcsv file path: {}\n".format(csv_file)
+                err_msg += "\terror line content: {}".format(line_data)
+                raise exception.FileFormatError(err_msg)
+            else:
+                data = {}
+                for index, parameter_name in enumerate(parameter_list):
+                    data[parameter_name] = line_data[index]
+
+                csv_content_list.append(data)
+
+    return csv_content_list
+
 def load_file(file_path):
-    file_suffix = os.path.splitext(file_path)[1]
+    if not os.path.isfile(file_path):
+        raise exception.FileNotFoundError("{} does not exist.".format(file_path))
+
+    file_suffix = os.path.splitext(file_path)[1].lower()
     if file_suffix == '.json':
         return _load_json_file(file_path)
     elif file_suffix in ['.yaml', '.yml']:
         return _load_yaml_file(file_path)
+    elif file_suffix == ".csv":
+        return _load_csv_file(file_path)
     else:
         # '' or other suffix
         err_msg = u"file is not in YAML/JSON format: {}".format(file_path)
@@ -550,6 +610,67 @@ def check_format(file_path, content):
         logging.error(err_msg)
         raise exception.FileFormatError(err_msg)
 
+def gen_cartesian_product(*args):
+    """ generate cartesian product for lists
+    @param
+        (list) args
+            [{"a": 1}, {"a": 2}],
+            [
+                {"x": 111, "y": 112},
+                {"x": 121, "y": 122}
+            ]
+    @return
+        cartesian product in list
+        [
+            {'a': 1, 'x': 111, 'y': 112},
+            {'a': 1, 'x': 121, 'y': 122},
+            {'a': 2, 'x': 111, 'y': 112},
+            {'a': 2, 'x': 121, 'y': 122}
+        ]
+    """
+    if not args:
+        return []
+    elif len(args) == 1:
+        return args[0]
+
+    product_list = []
+    for product_item_tuple in itertools.product(*args):
+        product_item_dict = {}
+        for item in product_item_tuple:
+            product_item_dict.update(item)
+
+        product_list.append(product_item_dict)
+
+    return product_list
+
+def gen_cartesian_product_parameters(parameters, testset_path):
+    """ parse parameters and generate cartesian product
+    @params
+        (list) parameters: parameter name and fetch method
+            e.g.
+                [
+                    {"user_agent": "Random"},
+                    {"app_version": "Sequential"}
+                ]
+        (str) testset_path: testset file path, used for locating csv file
+    @return cartesian product in list
+    """
+    parameters_content_list = []
+    for parameter in parameters:
+        parameter_name, fetch_method = list(parameter.items())[0]
+        parameter_file_path = os.path.join(
+            os.path.dirname(testset_path),
+            "{}.csv".format(parameter_name)
+        )
+        csv_content_list = load_file(parameter_file_path)
+
+        if fetch_method.lower() == "random":
+            random.shuffle(csv_content_list)
+
+        parameters_content_list.append(csv_content_list)
+
+    return gen_cartesian_product(*parameters_content_list)
+
 
 class TestcaseParser(object):
 

tests/data/app_version.csv

@@ -0,0 +1,4 @@
+app_version
+2.8.5
+2.8.6
+

tests/data/demo_parameters.yml

@@ -0,0 +1,26 @@
+- config:
+    name: "user management testset."
+    parameters:
+        - user_agent: Random
+        - app_version: Sequential
+    variables:
+        - user_agent: 'iOS/10.3'
+        - device_sn: ${gen_random_string(15)}
+        - os_platform: 'ios'
+        - app_version: '2.8.6'
+    request:
+        base_url: $BASE_URL
+        headers:
+            Content-Type: application/json
+            device_sn: $device_sn
+    output:
+        - token
+
+- test:
+    name: get token with $user_agent and $app_version
+    api: get_token($user_agent, $device_sn, $os_platform, $app_version)
+    extract:
+        - token: content.token
+    validate:
+        - "eq": ["status_code", 200]
+        - "len_eq": ["content.token", 16]

tests/data/user_agent.csv

@@ -0,0 +1,4 @@
+user_agent
+iOS/10.1
+iOS/10.2
+iOS/10.3

tests/data/username-password.csv

@@ -0,0 +1,4 @@
+username,password
+test1,111111
+test2,222222
+test3,333333

tests/test_runner.py

@@ -158,3 +158,11 @@ class TestRunner(ApiServerUnittest):
 
         test = testcases[2]["test"]
         self.assertTrue(self.test_runner._run_test(test))
+
+    def test_run_testset_with_parameters(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_parameters.yml')
+        result = run_suite_path(testcase_file_path)
+        self.assertTrue(result.success)
+        self.assertIn("token", result.output)
+        self.assertEqual(result.stat.total, 6)

tests/test_testcase.py

@@ -3,14 +3,16 @@ import time
 import unittest
 
 from httprunner import testcase
-from httprunner.exception import ApiNotFound, FileFormatError, ParamsError
+from httprunner.exception import (ApiNotFound, FileFormatError,
+                                  FileNotFoundError, ParamsError)
 
 
 class TestcaseParserUnittest(unittest.TestCase):
 
     def test_load_testcases_bad_filepath(self):
         testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo')
-        self.assertEqual(testcase.load_file(testcase_file_path), [])
+        with self.assertRaises(FileNotFoundError):
+            testcase.load_file(testcase_file_path)
 
     def test_load_json_testcases(self):
         testcase_file_path = os.path.join(
@@ -34,6 +36,111 @@ class TestcaseParserUnittest(unittest.TestCase):
         self.assertIn('url', test['request'])
         self.assertIn('method', test['request'])
 
+    def test_load_csv_file_one_parameter(self):
+        csv_file_path = os.path.join(
+            os.getcwd(), 'tests/data/user_agent.csv')
+        csv_content = testcase.load_file(csv_file_path)
+        self.assertEqual(
+            csv_content,
+            [
+                {'user_agent': 'iOS/10.1'},
+                {'user_agent': 'iOS/10.2'},
+                {'user_agent': 'iOS/10.3'}
+            ]
+        )
+
+    def test_load_csv_file_multiple_parameters(self):
+        csv_file_path = os.path.join(
+            os.getcwd(), 'tests/data/username-password.csv')
+        csv_content = testcase.load_file(csv_file_path)
+        self.assertEqual(
+            csv_content,
+            [
+                {'username': 'test1', 'password': '111111'},
+                {'username': 'test2', 'password': '222222'},
+                {'username': 'test3', 'password': '333333'}
+            ]
+        )
+
+    def test_cartesian_product_one(self):
+        parameters_content_list = [
+            [
+                {"a": 1},
+                {"a": 2}
+            ]
+        ]
+        product_list = testcase.gen_cartesian_product(*parameters_content_list)
+        self.assertEqual(
+            product_list,
+            [
+                {"a": 1},
+                {"a": 2}
+            ]
+        )
+
+    def test_cartesian_product_multiple(self):
+        parameters_content_list = [
+            [
+                {"a": 1},
+                {"a": 2}
+            ],
+            [
+                {"x": 111, "y": 112},
+                {"x": 121, "y": 122}
+            ]
+        ]
+        product_list = testcase.gen_cartesian_product(*parameters_content_list)
+        self.assertEqual(
+            product_list,
+            [
+                {'a': 1, 'x': 111, 'y': 112},
+                {'a': 1, 'x': 121, 'y': 122},
+                {'a': 2, 'x': 111, 'y': 112},
+                {'a': 2, 'x': 121, 'y': 122}
+            ]
+        )
+
+    def test_cartesian_product_empty(self):
+        parameters_content_list = []
+        product_list = testcase.gen_cartesian_product(*parameters_content_list)
+        self.assertEqual(product_list, [])
+
+    def test_gen_cartesian_product_parameters_one_to_one(self):
+        parameters = [
+            {"user_agent": "random"},
+            {"app_version": "sequential"}
+        ]
+        testset_path = os.path.join(
+            os.getcwd(),
+            "tests/data/demo_parameters.yml"
+        )
+        cartesian_product_parameters = testcase.gen_cartesian_product_parameters(
+            parameters,
+            testset_path
+        )
+        self.assertEqual(
+            len(cartesian_product_parameters),
+            6
+        )
+
+    def test_gen_cartesian_product_parameters_one_to_multiple(self):
+        parameters = [
+            {"user_agent": "random"},
+            {"username-password": "sequential"}
+        ]
+        testset_path = os.path.join(
+            os.getcwd(),
+            "tests/data/demo_parameters.yml"
+        )
+        cartesian_product_parameters = testcase.gen_cartesian_product_parameters(
+            parameters,
+            testset_path
+        )
+        self.assertEqual(
+            len(cartesian_product_parameters),
+            9
+        )
+
     def test_load_yaml_file_file_format_error(self):
         yaml_tmp_file = "tests/data/tmp.yml"
         # create empty yaml file