rename test folder to tests

tests/api_server.py

@@ -0,0 +1,189 @@
+import hashlib
+import json
+from functools import wraps
+
+from flask import Flask, make_response, request
+from ate import utils
+
+app = Flask(__name__)
+
+""" storage all users' data
+data structure:
+    users_dict = {
+        'uid1': {
+            'name': 'name1',
+            'password': 'pwd1'
+        },
+        'uid2': {
+            'name': 'name2',
+            'password': 'pwd2'
+        }
+    }
+"""
+users_dict = {}
+
+AUTHENTICATION = False
+TOKEN = "debugtalk"
+
+def validate_request(func):
+
+    @wraps(func)
+    def wrapper(*args, **kwds):
+        if not AUTHENTICATION:
+            return func(*args, **kwds)
+
+        try:
+            req_headers = request.headers
+            req_authorization = req_headers['Authorization']
+            random_str = req_headers['Random']
+            data = utils.handle_req_data(request.data)
+            authorization = utils.gen_md5(TOKEN, data, random_str)
+            assert authorization == req_authorization
+            return func(*args, **kwds)
+        except (KeyError, AssertionError):
+            result = {
+                'success': False,
+                'msg': "Authorization failed!"
+            }
+            response = make_response(json.dumps(result), 403)
+            response.headers["Content-Type"] = "application/json"
+            return response
+
+    return wrapper
+
+
+@app.route('/')
+@validate_request
+def index():
+    return "Hello World!"
+
+@app.route('/customize-response', methods=['POST'])
+@validate_request
+def get_customized_response():
+    expected_resp_json = request.get_json()
+    status_code = expected_resp_json.get('status_code', 200)
+    headers_dict = expected_resp_json.get('headers', {})
+    body = expected_resp_json.get('body', {})
+    response = make_response(json.dumps(body), status_code)
+
+    for header_key, header_value in headers_dict.items():
+        response.headers[header_key] = header_value
+
+    return response
+
+@app.route('/api/token')
+@validate_request
+def get_token():
+    result = {
+        'success': True,
+        'token': utils.gen_random_string(8)
+    }
+    response = make_response(json.dumps(result))
+    response.headers["Content-Type"] = "application/json"
+    return response
+
+@app.route('/api/users')
+@validate_request
+def get_users():
+    users_list = [user for uid, user in users_dict.items()]
+    users = {
+        'success': True,
+        'count': len(users_list),
+        'items': users_list
+    }
+    response = make_response(json.dumps(users))
+    response.headers["Content-Type"] = "application/json"
+    return response
+
+@app.route('/api/users', methods=['DELETE'])
+@validate_request
+def clear_users():
+    users_dict.clear()
+    result = {
+        'success': True
+    }
+    response = make_response(json.dumps(result))
+    response.headers["Content-Type"] = "application/json"
+    return response
+
+@app.route('/api/users/<int:uid>', methods=['POST'])
+@validate_request
+def create_user(uid):
+    user = request.get_json()
+    if uid not in users_dict:
+        result = {
+            'success': True,
+            'msg': "user created successfully."
+        }
+        status_code = 201
+        users_dict[uid] = user
+    else:
+        result = {
+            'success': False,
+            'msg': "user already existed."
+        }
+        status_code = 500
+
+    response = make_response(json.dumps(result), status_code)
+    response.headers["Content-Type"] = "application/json"
+    return response
+
+@app.route('/api/users/<int:uid>')
+@validate_request
+def get_user(uid):
+    user = users_dict.get(uid, {})
+    if user:
+        result = {
+            'success': True,
+            'data': user
+        }
+        status_code = 200
+    else:
+        result = {
+            'success': False,
+            'data': user
+        }
+        status_code = 404
+
+    response = make_response(json.dumps(result), status_code)
+    response.headers["Content-Type"] = "application/json"
+    return response
+
+@app.route('/api/users/<int:uid>', methods=['PUT'])
+@validate_request
+def update_user(uid):
+    user = users_dict.get(uid, {})
+    if user:
+        user = request.get_json()
+        success = True
+        status_code = 200
+    else:
+        success = False
+        status_code = 404
+
+    result = {
+        'success': success,
+        'data': user
+    }
+    response = make_response(json.dumps(result), status_code)
+    response.headers["Content-Type"] = "application/json"
+    return response
+
+@app.route('/api/users/<int:uid>', methods=['DELETE'])
+@validate_request
+def delete_user(uid):
+    user = users_dict.pop(uid, {})
+    if user:
+        success = True
+        status_code = 200
+    else:
+        success = False
+        status_code = 404
+
+    result = {
+        'success': success,
+        'data': user
+    }
+    response = make_response(json.dumps(result), status_code)
+    response.headers["Content-Type"] = "application/json"
+    return response

tests/base.py

@@ -0,0 +1,38 @@
+import multiprocessing
+import time
+import unittest
+
+from ate import utils
+from tests import api_server
+
+
+class ApiServerUnittest(unittest.TestCase):
+    """ Test case class that sets up an HTTP server which can be used within the tests
+    """
+
+    authentication = False
+
+    @classmethod
+    def setUpClass(cls):
+        api_server.AUTHENTICATION = cls.authentication
+        cls.api_server_process = multiprocessing.Process(
+            target=api_server.app.run
+        )
+        cls.api_server_process.start()
+        time.sleep(0.1)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.api_server_process.terminate()
+
+    def prepare_headers(self, data=""):
+        token = api_server.TOKEN
+        data = utils.handle_req_data(data)
+        random_str = utils.gen_random_string(5)
+        authorization = utils.gen_md5(token, data, random_str)
+
+        headers = {
+            'authorization': authorization,
+            'random': random_str
+        }
+        return headers

tests/data/custom_functions.py

@@ -0,0 +1,72 @@
+import hashlib
+import json
+import random
+import string
+import time
+
+try:
+    string_type = basestring
+    PYTHON_VERSION = 2
+    import urllib
+except NameError:
+    string_type = str
+    PYTHON_VERSION = 3
+    import urllib.parse as urllib
+
+
+def gen_random_string(str_len):
+    return ''.join(
+        random.choice(string.ascii_letters + string.digits) for _ in range(str_len))
+
+def gen_md5(*args):
+    args = [handle_req_data(item) for item in args]
+    return hashlib.md5("".join(args).encode('utf-8')).hexdigest()
+
+def handle_req_data(data):
+
+    if PYTHON_VERSION == 3 and isinstance(data, bytes):
+        # In Python3, convert bytes to str
+        data = data.decode('utf-8')
+
+    if not data:
+        return data
+
+    if isinstance(data, str):
+        # check if data in str can be converted to dict
+        try:
+            data = json.loads(data)
+        except ValueError:
+            pass
+
+    if isinstance(data, dict):
+        # sort data in dict with keys, then convert to str
+        data = json.dumps(data, sort_keys=True)
+
+    return data
+
+def gen_urlencode_str(**kargs):
+    urlencoded_str = ""
+    quote_times = int(kargs.pop("quote_times", 1))
+
+    for key, value in kargs.items():
+        urlencoded_str += key
+        urlencoded_str += "="
+        if value == "undefined":
+            urlencoded_str += "undefined"
+        else:
+            if isinstance(value, (dict, list)):
+                value = json.dumps(value)
+            elif isinstance(value, (int, float)):
+                value = str(value)
+
+            value_str = value.encode('utf-8')
+            for _ in range(quote_times):
+                value_str = urllib.quote_plus(value_str)
+            urlencoded_str += value_str
+
+        urlencoded_str += "&"
+
+    return urlencoded_str.strip("&")
+
+def get_timestamp():
+    return int(time.time() * 1000)

tests/data/demo_binds.yml

@@ -0,0 +1,42 @@
+register_variables:
+    variable_binds:
+        - TOKEN: "debugtalk"
+        - var: [1, 2, 3]
+        - data: {'name': 'user', 'password': '123456'}
+
+register_template_variables:
+    variable_binds:
+        - TOKEN: "debugtalk"
+        - token: $TOKEN
+
+bind_lambda_functions:
+    function_binds:
+        add_one: "lambda x: x + 1"
+        add_two_nums: "lambda x, y: x + y"
+    variable_binds:
+        - add1: ${add_one(2)}
+        - sum2nums: ${add_two_nums(2, 3)}
+
+bind_lambda_functions_with_import:
+    requires:
+        - random
+        - string
+        - hashlib
+    function_binds:
+        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
+        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
+    variable_binds:
+        - TOKEN: debugtalk
+        - random: ${gen_random_string(5)}
+        - data: "{'name': 'user', 'password': '123456'}"
+        - authorization: ${gen_md5($TOKEN, $data, $random)}
+
+bind_module_functions:
+    function_binds:
+        import_module_functions:
+            - tests.data.custom_functions
+    variable_binds:
+        - TOKEN: debugtalk
+        - random: ${gen_random_string(5)}
+        - data: "{'name': 'user', 'password': '123456'}"
+        - authorization: ${gen_md5($TOKEN, $data, $random)}

tests/data/demo_import_functions.yml

@@ -0,0 +1,41 @@
+- config:
+    name: "create user testsets."
+    import_module_functions:
+        - tests.data.custom_functions
+    variable_binds:
+        - TOKEN: debugtalk
+        - json: {}
+        - random: ${gen_random_string(5)}
+        - authorization: ${gen_md5($TOKEN, $json, $random)}
+
+- test:
+    name: create user which does not exist
+    variable_binds:
+        - json: {"name": "user", "password": "123456"}
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: $authorization
+            random: $random
+        json: $json
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
+
+- test:
+    name: create user which does not exist
+    variable_binds:
+        - json: {"name": "user", "password": "123456"}
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: $authorization
+            random: $random
+        json: $json
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

tests/data/demo_template_separate.yml

@@ -0,0 +1,52 @@
+
+- test:
+    name: create user which does not exist
+    requires:
+        - random
+        - string
+        - hashlib
+    function_binds:
+        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
+        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
+    variable_binds:
+        - TOKEN: debugtalk
+        - random: ${gen_random_string(5)}
+        - data: '{"name": "user", "password": "123456"}'
+        - authorization: ${gen_md5($TOKEN, $data, $random)}
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: $authorization
+            random: $random
+        data: $data
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
+
+- test:
+    name: create user which does exist
+    requires:
+        - random
+        - string
+        - hashlib
+    function_binds:
+        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
+        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
+    variable_binds:
+        - TOKEN: debugtalk
+        - random: ${gen_random_string(5)}
+        - data: '{"name": "user", "password": "123456"}'
+        - authorization: ${gen_md5($TOKEN, $data, $random)}
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: $authorization
+            random: $random
+        data: $data
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

tests/data/demo_template_sets.yml

@@ -0,0 +1,49 @@
+- config:
+    name: "create user testsets."
+    requires:
+        - random
+        - string
+        - hashlib
+    function_binds:
+        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
+        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
+    variable_binds:
+        - TOKEN: debugtalk
+        - data: ""
+        - random: ${gen_random_string(5)}
+        - authorization: ${gen_md5($TOKEN, $data, $random)}
+    request:
+        base_url: http://127.0.0.1:5000
+
+- test:
+    name: create user which does not exist
+    variable_binds:
+        - data: '{"name": "user", "password": "123456"}'
+    request:
+        url: /api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: $authorization
+            random: $random
+        data: $data
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
+
+- test:
+    name: create user which does exist
+    variable_binds:
+        - data: '{"name": "user", "password": "123456"}'
+        - expected_status_code: 500
+    request:
+        url: /api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: $authorization
+            random: $random
+        data: $data
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

tests/data/simple_demo_auth_hardcode.json

@@ -0,0 +1,46 @@
+[
+    {
+        "test": {
+            "name": "create user which does not exist",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/1000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json",
+                    "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
+                    "random": "A2dEx"
+                },
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
+            },
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 201},
+                {"check": "content.success", "comparator": "eq", "expected": true}
+            ]
+        }
+    },
+    {
+        "test": {
+            "name": "create user which existed",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/1000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json",
+                    "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
+                    "random": "A2dEx"
+                },
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
+            },
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 500},
+                {"check": "content.success", "comparator": "eq", "expected": false}
+            ]
+        }
+    }
+]

tests/data/simple_demo_auth_hardcode.yml

@@ -0,0 +1,31 @@
+- test:
+    name: create user which does not exist
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: a83de0ff8d2e896dbd8efb81ba14e17d
+            random: A2dEx
+        json:
+            name: "user1"
+            password: "123456"
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
+
+- test:
+    name: create user which existed
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: a83de0ff8d2e896dbd8efb81ba14e17d
+            random: A2dEx
+        json:
+            name: "user1"
+            password: "123456"
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

tests/data/simple_demo_no_auth.json

@@ -0,0 +1,43 @@
+[
+    {
+        "test": {
+            "name": "create user which does not exist",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/1000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json"
+                },
+                "cookies": {},
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
+            },
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 201},
+                {"check": "content.success", "comparator": "eq", "expected": true}
+            ]
+        }
+    },
+    {
+        "test": {
+            "name": "create user which existed",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/1000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json"
+                },
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
+            },
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 500},
+                {"check": "content.success", "comparator": "eq", "expected": false}
+            ]
+        }
+    }
+]

tests/data/simple_demo_no_auth.yml

@@ -0,0 +1,29 @@
+- test:
+    name: create user which does not exist
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+        json:
+            name: user1
+            password: 123456
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
+        - {"check": "headers.content-type", "comparator": "eq", "expected": "application/json"}
+
+- test:
+    name: create user which existed
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+        json:
+            name: user1
+            password: 123456
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}
+        - {"check": "headers.content-type", "comparator": "eq", "expected": "application/json"}

tests/test_apiserver.py

@@ -0,0 +1,137 @@
+import requests
+import random
+from tests.base import ApiServerUnittest
+
+class TestApiServer(ApiServerUnittest):
+    def setUp(self):
+        super(TestApiServer, self).setUp()
+        self.host = "http://127.0.0.1:5000"
+        self.api_client = requests.Session()
+        self.clear_users()
+
+    def tearDown(self):
+        super(TestApiServer, self).tearDown()
+
+    def clear_users(self):
+        url = "%s/api/users" % self.host
+        return self.api_client.delete(url)
+
+    def get_users(self):
+        url = "%s/api/users" % self.host
+        return self.api_client.get(url)
+
+    def create_user(self, uid, name, password):
+        url = "%s/api/users/%d" % (self.host, uid)
+        data = {
+            'name': name,
+            'password': password
+        }
+        return self.api_client.post(url, json=data)
+
+    def get_user(self, uid):
+        url = "%s/api/users/%d" % (self.host, uid)
+        return self.api_client.get(url)
+
+    def update_user(self, uid, name, password):
+        url = "%s/api/users/%d" % (self.host, uid)
+        data = {
+            'name': name,
+            'password': password
+        }
+        return self.api_client.put(url, json=data)
+
+    def delete_user(self, uid):
+        url = "%s/api/users/%d" % (self.host, uid)
+        return self.api_client.delete(url)
+
+    def test_clear_users(self):
+        resp = self.clear_users()
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(True, resp.json()['success'])
+
+    def test_create_user_not_existed(self):
+        resp = self.create_user(1000, 'user1', '123456')
+        self.assertEqual(201, resp.status_code)
+        self.assertEqual(True, resp.json()['success'])
+
+    def test_create_user_existed(self):
+        resp = self.create_user(1000, 'user1', '123456')
+        resp = self.create_user(1000, 'user1', '123456')
+        self.assertEqual(500, resp.status_code)
+
+    def test_get_users_empty(self):
+        resp = self.get_users()
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['count'], 0)
+
+    def test_get_users_not_empty(self):
+        resp = self.create_user(1000, 'user1', '123456')
+        resp = self.get_users()
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['count'], 1)
+
+        resp = self.create_user(1001, 'user2', '123456')
+        resp = self.get_users()
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['count'], 2)
+
+    def test_get_user_not_existed(self):
+        resp = self.get_user(1000)
+        self.assertEqual(404, resp.status_code)
+        self.assertEqual(resp.json()['success'], False)
+
+    def test_get_user_existed(self):
+        self.create_user(1000, 'user1', '123456')
+        resp = self.get_user(1000)
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['success'], True)
+
+    def test_update_user_not_existed(self):
+        resp = self.update_user(1000, 'user1', '123456')
+        self.assertEqual(404, resp.status_code)
+        self.assertEqual(resp.json()['success'], False)
+
+    def test_update_user_existed(self):
+        self.create_user(1000, 'user1', '123456')
+        resp = self.update_user(1000, 'user2', '123456')
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['data']['name'], 'user2')
+
+    def test_delete_user_not_existed(self):
+        resp = self.delete_user(1000)
+        self.assertEqual(404, resp.status_code)
+        self.assertEqual(resp.json()['success'], False)
+
+    def test_delete_user_existed(self):
+        self.create_user(1000, 'leo', '123456')
+        resp = self.delete_user(1000)
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['success'], True)
+
+    def test_get_customized_response_status_code(self):
+        status_code = random.randint(200, 511)
+        url = "%s/customize-response" % self.host
+        expected_response = {
+            'status_code': status_code,
+        }
+        resp = self.api_client.post(url, json=expected_response)
+        self.assertEqual(status_code, resp.status_code)
+
+    def test_get_customized_response_headers(self):
+        expected_response = {
+            'headers': {
+                'abc': 123,
+                'def': 456
+            }
+        }
+        url = "%s/customize-response" % self.host
+        resp = self.api_client.post(url, json=expected_response)
+        self.assertIn('abc', resp.headers)
+        self.assertIn('123', resp.headers['abc'])
+
+    def test_get_token(self):
+        url = "%s/api/token" % self.host
+        resp = self.api_client.get(url)
+        resp_json = resp.json()
+        self.assertTrue(resp_json["success"])
+        self.assertEqual(len(resp_json["token"]), 8)

tests/test_apiserver_v2.py

@@ -0,0 +1,158 @@
+import random
+import requests
+
+from tests.base import ApiServerUnittest
+
+
+class TestApiServerV2(ApiServerUnittest):
+
+    authentication = True
+
+    def setUp(self):
+        super(TestApiServerV2, self).setUp()
+        self.host = "http://127.0.0.1:5000"
+        self.api_client = requests.Session()
+        self.clear_users()
+
+    def tearDown(self):
+        super(TestApiServerV2, self).tearDown()
+
+    def test_index(self):
+        headers = self.prepare_headers()
+        resp = self.api_client.get(self.host, headers=headers)
+        self.assertEqual(200, resp.status_code)
+
+    def clear_users(self):
+        url = "%s/api/users" % self.host
+        return self.api_client.delete(url, headers=self.prepare_headers())
+
+    def get_users(self):
+        url = "%s/api/users" % self.host
+        return self.api_client.get(url, headers=self.prepare_headers())
+
+    def create_user(self, uid, name, password):
+        url = "%s/api/users/%d" % (self.host, uid)
+        data = {
+            'name': name,
+            'password': password
+        }
+        headers = self.prepare_headers(data)
+        return self.api_client.post(url, headers=headers, json=data)
+
+    def get_user(self, uid):
+        url = "%s/api/users/%d" % (self.host, uid)
+        return self.api_client.get(url, headers=self.prepare_headers())
+
+    def update_user(self, uid, name, password):
+        url = "%s/api/users/%d" % (self.host, uid)
+        data = {
+            'name': name,
+            'password': password
+        }
+        headers = self.prepare_headers(data)
+        return self.api_client.put(url, headers=headers, json=data)
+
+    def delete_user(self, uid):
+        url = "%s/api/users/%d" % (self.host, uid)
+        return self.api_client.delete(url, headers=self.prepare_headers())
+
+    def test_clear_users(self):
+        resp = self.clear_users()
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(True, resp.json()['success'])
+
+    def test_create_user_not_existed(self):
+        resp = self.create_user(1000, 'user1', '123456')
+        self.assertEqual(201, resp.status_code)
+        self.assertEqual(True, resp.json()['success'])
+
+    def test_create_user_existed(self):
+        resp = self.create_user(1000, 'user1', '123456')
+        resp = self.create_user(1000, 'user1', '123456')
+        self.assertEqual(500, resp.status_code)
+
+    def test_get_users_empty(self):
+        resp = self.get_users()
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['count'], 0)
+
+    def test_get_users_not_empty(self):
+        resp = self.create_user(1000, 'user1', '123456')
+        resp = self.get_users()
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['count'], 1)
+
+        resp = self.create_user(1001, 'user2', '123456')
+        resp = self.get_users()
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['count'], 2)
+
+    def test_get_user_not_existed(self):
+        resp = self.get_user(1000)
+        self.assertEqual(404, resp.status_code)
+        self.assertEqual(resp.json()['success'], False)
+
+    def test_get_user_existed(self):
+        self.create_user(1000, 'user1', '123456')
+        resp = self.get_user(1000)
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['success'], True)
+
+    def test_update_user_not_existed(self):
+        resp = self.update_user(1000, 'user1', '123456')
+        self.assertEqual(404, resp.status_code)
+        self.assertEqual(resp.json()['success'], False)
+
+    def test_update_user_existed(self):
+        self.create_user(1000, 'user1', '123456')
+        resp = self.update_user(1000, 'user2', '123456')
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['data']['name'], 'user2')
+
+    def test_delete_user_not_existed(self):
+        resp = self.delete_user(1000)
+        self.assertEqual(404, resp.status_code)
+        self.assertEqual(resp.json()['success'], False)
+
+    def test_delete_user_existed(self):
+        self.create_user(1000, 'leo', '123456')
+        resp = self.delete_user(1000)
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual(resp.json()['success'], True)
+
+    def test_get_customized_response_status_code(self):
+        status_code = random.randint(200, 511)
+        url = "%s/customize-response" % self.host
+        expected_response = {
+            'status_code': status_code,
+        }
+        resp = self.api_client.post(
+            url,
+            headers=self.prepare_headers(expected_response),
+            json=expected_response
+        )
+        self.assertEqual(status_code, resp.status_code)
+
+    def test_get_customized_response_headers(self):
+        expected_response = {
+            'headers': {
+                'abc': 123,
+                'def': 456
+            }
+        }
+        url = "%s/customize-response" % self.host
+        resp = self.api_client.post(
+            url,
+            headers=self.prepare_headers(expected_response),
+            json=expected_response
+        )
+        self.assertIn('abc', resp.headers)
+        self.assertIn('123', resp.headers['abc'])
+
+    def test_get_token(self):
+        url = "%s/api/token" % self.host
+        headers = self.prepare_headers()
+        resp = self.api_client.get(url, headers=headers)
+        resp_json = resp.json()
+        self.assertTrue(resp_json["success"])
+        self.assertEqual(len(resp_json["token"]), 8)

tests/test_client.py

@@ -0,0 +1,36 @@
+from ate.client import HttpSession
+from tests.base import ApiServerUnittest
+
+class TestHttpClient(ApiServerUnittest):
+    def setUp(self):
+        super(TestHttpClient, self).setUp()
+        self.host = "http://127.0.0.1:5000"
+        self.api_client = HttpSession(self.host)
+        self.clear_users()
+
+    def tearDown(self):
+        super(TestHttpClient, self).tearDown()
+
+    def clear_users(self):
+        url = "%s/api/users" % self.host
+        return self.api_client.delete(url)
+
+    def test_request_with_full_url(self):
+        url = "%s/api/users/1000" % self.host
+        data = {
+            'name': 'user1',
+            'password': '123456'
+        }
+        resp = self.api_client.post(url, json=data)
+        self.assertEqual(201, resp.status_code)
+        self.assertEqual(True, resp.json()['success'])
+
+    def test_request_without_base_url(self):
+        url = "/api/users/1000"
+        data = {
+            'name': 'user1',
+            'password': '123456'
+        }
+        resp = self.api_client.post(url, json=data)
+        self.assertEqual(201, resp.status_code)
+        self.assertEqual(True, resp.json()['success'])

tests/test_context.py

@@ -0,0 +1,218 @@
+import os
+import unittest
+
+from ate import utils, runner
+from ate.context import Context
+
+
+class VariableBindsUnittest(unittest.TestCase):
+
+    def setUp(self):
+        self.context = Context()
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml')
+        self.testcases = utils.load_testcases(testcase_file_path)
+
+    def test_context_register_variables(self):
+        # testcase in JSON format
+        testcase1 = {
+            "variable_binds": [
+                {"TOKEN": "debugtalk"},
+                {"var": [1, 2, 3]},
+                {"data": {'name': 'user', 'password': '123456'}}
+            ]
+        }
+        # testcase in YAML format
+        testcase2 = self.testcases["register_variables"]
+
+        for testcase in [testcase1, testcase2]:
+            variable_binds = testcase['variable_binds']
+            self.context.register_variables_config(variable_binds)
+
+            context_variables = self.context._get_evaluated_testcase_variables()
+            self.assertIn("TOKEN", context_variables)
+            self.assertEqual(context_variables["TOKEN"], "debugtalk")
+            self.assertIn("var", context_variables)
+            self.assertEqual(context_variables["var"], [1, 2, 3])
+            self.assertIn("data", context_variables)
+            self.assertEqual(
+                context_variables["data"],
+                {'name': 'user', 'password': '123456'}
+            )
+
+    def test_context_register_template_variables(self):
+        testcase1 = {
+            "variable_binds": [
+                {"GLOBAL_TOKEN": "debugtalk"},
+                {"token": "$GLOBAL_TOKEN"}
+            ]
+        }
+        testcase2 = self.testcases["register_template_variables"]
+
+        for testcase in [testcase1, testcase2]:
+            variable_binds = testcase['variable_binds']
+            self.context.register_variables_config(variable_binds)
+
+            context_variables = self.context._get_evaluated_testcase_variables()
+            self.assertIn("GLOBAL_TOKEN", context_variables)
+            self.assertEqual(context_variables["GLOBAL_TOKEN"], "debugtalk")
+            self.assertIn("token", context_variables)
+            self.assertEqual(context_variables["token"], "debugtalk")
+
+    def test_context_bind_lambda_functions(self):
+        testcase1 = {
+            "function_binds": {
+                "add_one": lambda x: x + 1,
+                "add_two_nums": lambda x, y: x + y
+            },
+            "variable_binds": [
+                {"add1": "${add_one(2)}"},
+                {"sum2nums": "${add_two_nums(2,3)}"}
+            ]
+        }
+        testcase2 = self.testcases["bind_lambda_functions"]
+
+        for testcase in [testcase1, testcase2]:
+            function_binds = testcase.get('function_binds', {})
+            self.context.bind_functions(function_binds)
+
+            variable_binds = testcase['variable_binds']
+            self.context.register_variables_config(variable_binds)
+
+            context_variables = self.context._get_evaluated_testcase_variables()
+            self.assertIn("add1", context_variables)
+            self.assertEqual(context_variables["add1"], 3)
+            self.assertIn("sum2nums", context_variables)
+            self.assertEqual(context_variables["sum2nums"], 5)
+
+    def test_context_bind_lambda_functions_with_import(self):
+        testcase1 = {
+            "requires": ["random", "string", "hashlib"],
+            "function_binds": {
+                "gen_random_string": "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))",
+                "gen_md5": "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
+            },
+            "variable_binds": [
+                {"TOKEN": "debugtalk"},
+                {"random": "${gen_random_string(5)}"},
+                {"data": '{"name": "user", "password": "123456"}'},
+                {"authorization": "${gen_md5($TOKEN, $data, $random)}"}
+            ]
+        }
+        testcase2 = self.testcases["bind_lambda_functions_with_import"]
+
+        for testcase in [testcase1, testcase2]:
+            requires = testcase.get('requires', [])
+            self.context.import_requires(requires)
+
+            function_binds = testcase.get('function_binds', {})
+            self.context.bind_functions(function_binds)
+
+            variable_binds = testcase['variable_binds']
+            self.context.register_variables_config(variable_binds)
+            context_variables = self.context._get_evaluated_testcase_variables()
+
+            self.assertIn("TOKEN", context_variables)
+            TOKEN = context_variables["TOKEN"]
+            self.assertEqual(TOKEN, "debugtalk")
+            self.assertIn("random", context_variables)
+            self.assertIsInstance(context_variables["random"], str)
+            self.assertEqual(len(context_variables["random"]), 5)
+            random = context_variables["random"]
+            self.assertIn("data", context_variables)
+            data = context_variables["data"]
+            self.assertIn("authorization", context_variables)
+            self.assertEqual(len(context_variables["authorization"]), 32)
+            authorization = context_variables["authorization"]
+            self.assertEqual(utils.gen_md5(TOKEN, data, random), authorization)
+
+    def test_import_module_functions(self):
+        testcase1 = {
+            "import_module_functions": ["tests.data.custom_functions"],
+            "variable_binds": [
+                {"TOKEN": "debugtalk"},
+                {"random": "${gen_random_string(5)}"},
+                {"data": '{"name": "user", "password": "123456"}'},
+                {"authorization": "${gen_md5($TOKEN, $data, $random)}"}
+            ]
+        }
+        testcase2 = self.testcases["bind_module_functions"]
+
+        for testcase in [testcase1, testcase2]:
+            module_functions = testcase.get('import_module_functions', [])
+            self.context.import_module_functions(module_functions)
+
+            variable_binds = testcase['variable_binds']
+            self.context.register_variables_config(variable_binds)
+            context_variables = self.context._get_evaluated_testcase_variables()
+
+            self.assertIn("TOKEN", context_variables)
+            TOKEN = context_variables["TOKEN"]
+            self.assertEqual(TOKEN, "debugtalk")
+            self.assertIn("random", context_variables)
+            self.assertIsInstance(context_variables["random"], str)
+            self.assertEqual(len(context_variables["random"]), 5)
+            random = context_variables["random"]
+            self.assertIn("data", context_variables)
+            data = context_variables["data"]
+            self.assertIn("authorization", context_variables)
+            self.assertEqual(len(context_variables["authorization"]), 32)
+            authorization = context_variables["authorization"]
+            self.assertEqual(utils.gen_md5(TOKEN, data, random), authorization)
+
+    def test_get_parsed_request(self):
+        test_runner = runner.Runner()
+        testcase = {
+            "import_module_functions": ["tests.data.custom_functions"],
+            "variable_binds": [
+                {"TOKEN": "debugtalk"},
+                {"random": "${gen_random_string(5)}"},
+                {"data": '{"name": "user", "password": "123456"}'},
+                {"authorization": "${gen_md5($TOKEN, $data, $random)}"}
+            ],
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/1000",
+                "method": "POST",
+                "headers": {
+                    "Content-Type": "application/json",
+                    "authorization": "$authorization",
+                    "random": "$random"
+                },
+                "data": "$data"
+            }
+        }
+        test_runner.init_config(testcase, level="testcase")
+        parsed_request = test_runner.context.get_parsed_request()
+        self.assertIn("authorization", parsed_request["headers"])
+        self.assertEqual(len(parsed_request["headers"]["authorization"]), 32)
+        self.assertIn("random", parsed_request["headers"])
+        self.assertEqual(len(parsed_request["headers"]["random"]), 5)
+        self.assertIn("data", parsed_request)
+        self.assertEqual(parsed_request["data"], testcase["variable_binds"][2]["data"])
+
+    def test_get_eval_value(self):
+        self.context.testcase_variables_mapping = {
+            "str_1": "str_value1",
+            "str_2": "str_value2"
+        }
+        self.assertEqual(self.context.get_eval_value("$str_1"), "str_value1")
+        self.assertEqual(self.context.get_eval_value("$str_2"), "str_value2")
+        self.assertEqual(
+            self.context.get_eval_value(["$str_1", "str3"]),
+            ["str_value1", "str3"]
+        )
+        self.assertEqual(
+            self.context.get_eval_value({"key": "$str_1"}),
+            {"key": "str_value1"}
+        )
+
+        import random, string
+        self.context.testcase_config["functions"]["gen_random_string"] = \
+            lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) \
+                for _ in range(str_len))
+        result = self.context.get_eval_value("${gen_random_string(5)}")
+        self.assertEqual(len(result), 5)
+
+        add_two_nums = lambda a, b=1: a + b
+        self.context.testcase_config["functions"]["add_two_nums"] = add_two_nums
+        self.assertEqual(self.context.get_eval_value("${add_two_nums(1)}"), 2)
+        self.assertEqual(self.context.get_eval_value("${add_two_nums(1, 2)}"), 3)

tests/test_response.py

@@ -0,0 +1,210 @@
+import requests
+from ate import response, exception
+from tests.base import ApiServerUnittest
+
+class TestResponse(ApiServerUnittest):
+
+    def test_parse_response_object_json(self):
+        url = "http://127.0.0.1:5000/api/users"
+        resp = requests.get(url)
+        resp_obj = response.ResponseObject(resp)
+        parsed_dict = resp_obj.parsed_dict()
+        self.assertIn('status_code', parsed_dict)
+        self.assertIn('headers', parsed_dict)
+        self.assertIn('body', parsed_dict)
+        self.assertIn('Content-Type', parsed_dict['headers'])
+        self.assertIn('Content-Length', parsed_dict['headers'])
+        self.assertIn('success', parsed_dict['body'])
+
+    def test_parse_response_object_text(self):
+        url = "http://127.0.0.1:5000/"
+        resp = requests.get(url)
+        resp_obj = response.ResponseObject(resp)
+        parsed_dict = resp_obj.parsed_dict()
+        self.assertIn('status_code', parsed_dict)
+        self.assertIn('headers', parsed_dict)
+        self.assertIn('body', parsed_dict)
+        self.assertIn('Content-Type', parsed_dict['headers'])
+        self.assertIn('Content-Length', parsed_dict['headers'])
+        self.assertTrue(str, type(parsed_dict['body']))
+
+    def test_extract_response_json(self):
+        resp = requests.post(
+            url="http://127.0.0.1:5000/customize-response",
+            json={
+                'headers': {
+                    'Content-Type': "application/json"
+                },
+                'body': {
+                    'success': False,
+                    "person": {
+                        "name": {
+                            "first_name": "Leo",
+                            "last_name": "Lee",
+                        },
+                        "age": 29,
+                        "cities": ["Guangzhou", "Shenzhen"]
+                    }
+                }
+            }
+        )
+
+        extract_binds = {
+            "resp_status_code": "status_code",
+            "resp_headers_content_type": "headers.content-type",
+            "resp_content_body_success": "body.success",
+            "resp_content_content_success": "content.success",
+            "resp_content_text_success": "text.success",
+            "resp_content_person_first_name": "content.person.name.first_name",
+            "resp_content_cities_1": "content.person.cities.1"
+        }
+        resp_obj = response.ResponseObject(resp)
+        extract_binds_dict = resp_obj.extract_response(extract_binds)
+
+        self.assertEqual(
+            extract_binds_dict["resp_status_code"],
+            200
+        )
+        self.assertEqual(
+            extract_binds_dict["resp_headers_content_type"],
+            "application/json"
+        )
+        self.assertEqual(
+            extract_binds_dict["resp_content_content_success"],
+            False
+        )
+        self.assertEqual(
+            extract_binds_dict["resp_content_text_success"],
+            False
+        )
+        self.assertEqual(
+            extract_binds_dict["resp_content_person_first_name"],
+            "Leo"
+        )
+        self.assertEqual(
+            extract_binds_dict["resp_content_cities_1"],
+            "Shenzhen"
+        )
+
+    def test_extract_response_fail(self):
+        resp = requests.post(
+            url="http://127.0.0.1:5000/customize-response",
+            json={
+                'headers': {
+                    'Content-Type': "application/json"
+                },
+                'body': {
+                    'success': False,
+                    "person": {
+                        "name": {
+                            "first_name": "Leo",
+                            "last_name": "Lee",
+                        },
+                        "age": 29,
+                        "cities": ["Guangzhou", "Shenzhen"]
+                    }
+                }
+            }
+        )
+
+        extract_binds = {
+            "resp_content_dict_key_error": "content.not_exist"
+        }
+        resp_obj = response.ResponseObject(resp)
+
+        with self.assertRaises(exception.ParamsError):
+            resp_obj.extract_response(extract_binds)
+
+        extract_binds = {
+            "resp_content_list_index_error": "content.person.cities.3"
+        }
+        resp_obj = response.ResponseObject(resp)
+
+        with self.assertRaises(exception.ParamsError):
+            resp_obj.extract_response(extract_binds)
+
+    def test_extract_response_json_string(self):
+        resp = requests.post(
+            url="http://127.0.0.1:5000/customize-response",
+            json={
+                'headers': {
+                    'Content-Type': "application/json"
+                },
+                'body': "abc"
+            }
+        )
+
+        extract_binds = {
+            "resp_content_body": "content"
+        }
+        resp_obj = response.ResponseObject(resp)
+
+        extract_binds_dict = resp_obj.extract_response(extract_binds)
+        self.assertEqual(
+            extract_binds_dict["resp_content_body"],
+            "abc"
+        )
+
+    def test_validate(self):
+        url = "http://127.0.0.1:5000/"
+        resp = requests.get(url)
+        resp_obj = response.ResponseObject(resp)
+
+        validators = [
+            {"check": "resp_status_code", "comparator": "eq", "expected": 201},
+            {"check": "resp_body_success", "comparator": "eq", "expected": True}
+        ]
+        variables_mapping = {
+            "resp_status_code": 200,
+            "resp_body_success": True
+        }
+
+        diff_content_list = resp_obj.validate(validators, variables_mapping)
+        self.assertFalse(resp_obj.success)
+        self.assertEqual(
+            diff_content_list,
+            [
+                {
+                    "check": "resp_status_code",
+                    "comparator": "eq", "expected": 201, "value": 200
+                }
+            ]
+        )
+
+        validators = [
+            {"check": "resp_status_code", "comparator": "eq", "expected": 201},
+            {"check": "resp_body_success", "comparator": "eq", "expected": True}
+        ]
+        variables_mapping = {
+            "resp_status_code": 201,
+            "resp_body_success": True
+        }
+
+        diff_content_list = resp_obj.validate(validators, variables_mapping)
+        self.assertTrue(resp_obj.success)
+        self.assertEqual(diff_content_list, [])
+
+    def test_validate_exception(self):
+        url = "http://127.0.0.1:5000/"
+        resp = requests.get(url)
+        resp_obj = response.ResponseObject(resp)
+
+        # expected value missed in validators
+        validators = [
+            {"check": "status_code", "comparator": "eq", "expected": 201},
+            {"check": "body_success", "comparator": "eq"}
+        ]
+        variables_mapping = {}
+        with self.assertRaises(exception.ParamsError):
+            resp_obj.validate(validators, variables_mapping)
+
+        # expected value missed in variables mapping
+        validators = [
+            {"check": "resp_status_code", "comparator": "eq", "expected": 201},
+            {"check": "body_success", "comparator": "eq"}
+        ]
+        variables_mapping = {
+            "resp_status_code": 200
+        }
+        with self.assertRaises(exception.ParamsError):
+            resp_obj.validate(validators, variables_mapping)

tests/test_runner.py

@@ -0,0 +1,97 @@
+import os
+import requests
+from ate import runner, exception, utils
+from tests.base import ApiServerUnittest
+
+class TestRunner(ApiServerUnittest):
+
+    def setUp(self):
+        self.test_runner = runner.Runner()
+        self.clear_users()
+
+    def clear_users(self):
+        url = "http://127.0.0.1:5000/api/users"
+        return requests.delete(url)
+
+    def test_run_single_testcase_yaml_success(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+        testcases = utils.load_testcases(testcase_file_path)
+        testcase = testcases[0]["test"]
+        success, _ = self.test_runner.run_test(testcase)
+        self.assertTrue(success)
+
+    def test_run_single_testcase_json_success(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.json')
+        testcases = utils.load_testcases(testcase_file_path)
+        testcase = testcases[0]["test"]
+        success, _ = self.test_runner.run_test(testcase)
+        self.assertTrue(success)
+
+    def test_run_single_testcase_fail(self):
+        testcase = {
+            "name": "create user which does not exist",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/1000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json"
+                },
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
+            },
+            "extract_binds": {
+                "resp_status_code": "status_code",
+                "resp_body_success": "content.success",
+                "resp_headers_contenttype": "headers.content-type"
+            },
+            "validators": [
+                {"check": "resp_status_code", "comparator": "eq", "expected": 200},
+                {"check": "resp_body_success", "comparator": "eq", "expected": False},
+                {"check": "resp_headers_contenttype", "comparator": "eq", "expected": "html/text"}
+            ]
+        }
+
+        success, diff_content_list = self.test_runner.run_test(testcase)
+        self.assertFalse(success)
+        self.assertEqual(
+            diff_content_list[0],
+            {"check": "resp_status_code", "comparator": "eq", "expected": 200, 'value': 201}
+        )
+        self.assertEqual(
+            diff_content_list[1],
+            {"check": "resp_body_success", "comparator": "eq", "expected": False, 'value': True}
+        )
+        self.assertEqual(
+            diff_content_list[2],
+            {"check": "resp_headers_contenttype", "comparator": "eq", "expected": "html/text", 'value': "application/json"}
+        )
+
+    def test_run_testset_json_success(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.json')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results, [(True, []), (True, [])])
+
+    def test_run_testsets_json_success(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.json')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testsets(testsets)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, []), (True, [])])
+
+    def test_run_testset_yaml_success(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results, [(True, []), (True, [])])
+
+    def test_run_testsets_yaml_success(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testsets(testsets)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, []), (True, [])])

tests/test_runner_v2.py

@@ -0,0 +1,105 @@
+import os
+import requests
+from ate import runner, exception, utils
+from tests.base import ApiServerUnittest
+
+class TestRunnerV2(ApiServerUnittest):
+
+    authentication = True
+
+    def setUp(self):
+        self.test_runner = runner.Runner()
+        self.clear_users()
+
+    def clear_users(self):
+        url = "http://127.0.0.1:5000/api/users"
+        return requests.delete(url, headers=self.prepare_headers())
+
+    def test_run_single_testcase_yaml(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.yml')
+        testcases = utils.load_testcases(testcase_file_path)
+        testcase = testcases[0]["test"]
+        success, _ = self.test_runner.run_test(testcase)
+        self.assertTrue(success)
+
+    def test_run_single_testcase_json(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.json')
+        testcases = utils.load_testcases(testcase_file_path)
+        testcase = testcases[0]["test"]
+        success, _ = self.test_runner.run_test(testcase)
+        self.assertTrue(success)
+
+    def test_run_testset_auth_yaml(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results, [(True, []), (True, [])])
+
+    def test_run_testsets_auth_yaml(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testsets(testsets)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, []), (True, [])])
+
+    def test_run_testset_auth_json(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.json')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results, [(True, []), (True, [])])
+
+    def test_run_testsets_auth_json(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.json')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testsets(testsets)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, []), (True, [])])
+
+    def test_run_testcase_template_yaml(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_template_separate.yml')
+        testcases = utils.load_testcases(testcase_file_path)
+        success, _ = self.test_runner.run_test(testcases[0]["test"])
+        self.assertTrue(success)
+        success, _ = self.test_runner.run_test(testcases[1]["test"])
+        self.assertTrue(success)
+
+    def test_run_testset_template_yaml(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_template_sets.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results, [(True, []), (True, [])])
+
+    def test_run_testsets_template_yaml(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_template_sets.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testsets(testsets)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, []), (True, [])])
+
+    def test_run_testset_template_import_functions(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_import_functions.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results, [(True, []), (True, [])])
+
+    def test_run_testsets_template_import_functions(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_import_functions.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testsets(testsets)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, []), (True, [])])

tests/test_task.py

@@ -0,0 +1,30 @@
+import os
+import random
+import requests
+from tests.base import ApiServerUnittest
+from ate import task, utils
+
+class TestMain(ApiServerUnittest):
+
+    def setUp(self):
+        self.clear_users()
+
+    def clear_users(self):
+        url = "http://127.0.0.1:5000/api/users"
+        return requests.delete(url)
+
+    def test_create_suite(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        suite = task.create_suite(testsets[0])
+        self.assertEqual(suite.countTestCases(), 2)
+        for testcase in suite:
+            self.assertIsInstance(testcase, task.ApiTestCase)
+
+    def test_create_task(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+        task_suite = task.create_task(testcase_file_path)
+        self.assertEqual(task_suite.countTestCases(), 2)
+        for suite in task_suite:
+            for testcase in suite:
+                self.assertIsInstance(testcase, task.ApiTestCase)

tests/test_testcase.py

@@ -0,0 +1,120 @@
+import unittest
+
+from ate.testcase import parse_template, parse_content_with_variables
+from ate import exception
+
+
+class TestcaseParserUnittest(unittest.TestCase):
+
+    def setUp(self):
+        self.variables_binds = {
+            "uid": "1000",
+            "random": "A2dEx",
+            "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
+            "json": {
+                "name": "user1",
+                "password": "123456"
+            },
+            "expected_status": 201,
+            "expected_success": True
+        }
+
+    def test_parse_testcase_template(self):
+        testcase = {
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/$uid",
+                "method": "POST",
+                "headers": {
+                    "Content-Type": "application/json",
+                    "authorization": "$authorization",
+                    "random": "$random"
+                },
+                "body": "$json"
+            },
+            "response": {
+                "status_code": "$expected_status",
+                "headers": {
+                    "Content-Type": "application/json"
+                },
+                "body": {
+                    "success": "$expected_success",
+                    "msg": "user created successfully."
+                }
+            }
+        }
+        parsed_testcase = parse_template(testcase, self.variables_binds)
+
+        self.assertEqual(
+            parsed_testcase["request"]["url"],
+            "http://127.0.0.1:5000/api/users/%s" % self.variables_binds["uid"]
+        )
+        self.assertEqual(
+            parsed_testcase["request"]["headers"]["authorization"],
+            self.variables_binds["authorization"]
+        )
+        self.assertEqual(
+            parsed_testcase["request"]["headers"]["random"],
+            self.variables_binds["random"]
+        )
+        self.assertEqual(
+            parsed_testcase["request"]["body"],
+            self.variables_binds["json"]
+        )
+        self.assertEqual(
+            parsed_testcase["response"]["status_code"],
+            self.variables_binds["expected_status"]
+        )
+        self.assertEqual(
+            parsed_testcase["response"]["body"]["success"],
+            self.variables_binds["expected_success"]
+        )
+
+    def test_parse_testcase_template_miss_bind_variable(self):
+        testcase = {
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/$uid",
+                "method": "$method"
+            }
+        }
+        with self.assertRaises(exception.ParamsError):
+            parse_template(testcase, self.variables_binds)
+
+    def test_parse_testcase_with_new_variable_binds(self):
+        testcase = {
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/$uid",
+                "method": "$method"
+            }
+        }
+        new_variable_binds = {
+            "method": "GET"
+        }
+        self.variables_binds.update(new_variable_binds)
+        parsed_testcase = parse_template(testcase, self.variables_binds)
+
+        self.assertEqual(
+            parsed_testcase["request"]["method"],
+            new_variable_binds["method"]
+        )
+
+    def test_parse_content_with_variables(self):
+        content = "$var"
+        variables_binds = {
+            "var": "abc"
+        }
+        result = parse_content_with_variables(content, variables_binds)
+        self.assertEqual(result, "abc")
+
+        content = "123$var/456"
+        variables_binds = {
+            "var": "abc"
+        }
+        result = parse_content_with_variables(content, variables_binds)
+        self.assertEqual(result, "123abc/456")
+
+        content = "$var1"
+        variables_binds = {
+            "var2": "abc"
+        }
+        with self.assertRaises(exception.ParamsError):
+            parse_content_with_variables(content, variables_binds)

tests/test_utils.py

@@ -0,0 +1,294 @@
+import os
+from ate import utils
+from ate import exception
+from tests.base import ApiServerUnittest
+
+class TestUtils(ApiServerUnittest):
+
+    def test_load_testcases_bad_filepath(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo')
+        with self.assertRaises(exception.ParamsError):
+            utils.load_testcases(testcase_file_path)
+
+    def test_load_json_testcases(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/simple_demo_no_auth.json')
+        testcases = utils.load_testcases(testcase_file_path)
+        self.assertEqual(len(testcases), 2)
+        testcase = testcases[0]["test"]
+        self.assertIn('name', testcase)
+        self.assertIn('request', testcase)
+        self.assertIn('url', testcase['request'])
+        self.assertIn('method', testcase['request'])
+
+    def test_load_yaml_testcases(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+        testcases = utils.load_testcases(testcase_file_path)
+        self.assertEqual(len(testcases), 2)
+        testcase = testcases[0]["test"]
+        self.assertIn('name', testcase)
+        self.assertIn('request', testcase)
+        self.assertIn('url', testcase['request'])
+        self.assertIn('method', testcase['request'])
+
+    def test_load_foler_files(self):
+        folder = os.path.join(os.getcwd(), 'tests')
+        files = utils.load_foler_files(folder)
+        file1 = os.path.join(os.getcwd(), 'tests', 'test_utils.py')
+        file2 = os.path.join(os.getcwd(), 'tests', 'data', 'demo_binds.yml')
+        self.assertIn(file1, files)
+        self.assertIn(file2, files)
+
+    def test_load_testcases_by_path_files(self):
+        testsets_list = []
+
+        # absolute file path
+        path = os.path.join(
+            os.getcwd(), 'tests/data/simple_demo_no_auth.json')
+        testset_list = utils.load_testcases_by_path(path)
+        self.assertEqual(len(testset_list), 1)
+        self.assertEqual(len(testset_list[0]["testcases"]), 2)
+        testsets_list.extend(testset_list)
+
+        # relative file path
+        path = 'tests/data/simple_demo_no_auth.yml'
+        testset_list = utils.load_testcases_by_path(path)
+        self.assertEqual(len(testset_list), 1)
+        self.assertEqual(len(testset_list[0]["testcases"]), 2)
+        testsets_list.extend(testset_list)
+
+        # list/set container with file(s)
+        path = [
+            os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.json'),
+            'tests/data/simple_demo_no_auth.yml'
+        ]
+        testset_list = utils.load_testcases_by_path(path)
+        self.assertEqual(len(testset_list), 2)
+        self.assertEqual(len(testset_list[0]["testcases"]), 2)
+        self.assertEqual(len(testset_list[1]["testcases"]), 2)
+        testsets_list.extend(testset_list)
+        self.assertEqual(len(testsets_list), 4)
+
+        for testset in testsets_list:
+            for testcase in testset["testcases"]:
+                self.assertIn('name', testcase)
+                self.assertIn('request', testcase)
+                self.assertIn('url', testcase['request'])
+                self.assertIn('method', testcase['request'])
+
+    def test_load_testcases_by_path_folder(self):
+        # absolute folder path
+        path = os.path.join(os.getcwd(), 'tests/data')
+        testset_list_1 = utils.load_testcases_by_path(path)
+        self.assertGreater(len(testset_list_1), 6)
+
+        # relative folder path
+        path = 'tests/data/'
+        testset_list_2 = utils.load_testcases_by_path(path)
+        self.assertEqual(len(testset_list_1), len(testset_list_2))
+
+        # list/set container with file(s)
+        path = [
+            os.path.join(os.getcwd(), 'tests/data'),
+            'tests/data/'
+        ]
+        testset_list_3 = utils.load_testcases_by_path(path)
+        self.assertEqual(len(testset_list_3), 2 * len(testset_list_1))
+
+    def test_load_testcases_by_path_not_exist(self):
+        # absolute folder path
+        path = os.path.join(os.getcwd(), 'tests/data_not_exist')
+        testset_list_1 = utils.load_testcases_by_path(path)
+        self.assertEqual(testset_list_1, [])
+
+        # relative folder path
+        path = 'tests/data_not_exist'
+        testset_list_2 = utils.load_testcases_by_path(path)
+        self.assertEqual(testset_list_2, [])
+
+        # list/set container with file(s)
+        path = [
+            os.path.join(os.getcwd(), 'tests/data_not_exist'),
+            'tests/data_not_exist/'
+        ]
+        testset_list_3 = utils.load_testcases_by_path(path)
+        self.assertEqual(testset_list_3, [])
+
+    def test_is_variable(self):
+        content = "$var"
+        self.assertTrue(utils.is_variable(content))
+        content = "$var123"
+        self.assertTrue(utils.is_variable(content))
+        content = "$var_name"
+        self.assertTrue(utils.is_variable(content))
+        content = "var"
+        self.assertFalse(utils.is_variable(content))
+        content = "a$var"
+        self.assertFalse(utils.is_variable(content))
+        content = "$v ar"
+        self.assertFalse(utils.is_variable(content))
+        content = " "
+        self.assertFalse(utils.is_variable(content))
+        content = "$abc*"
+        self.assertFalse(utils.is_variable(content))
+
+    def test_parse_variable(self):
+        content = "$var"
+        self.assertEqual(utils.parse_variable(content), "var")
+        content = "$var123"
+        self.assertEqual(utils.parse_variable(content), "var123")
+        content = "$var_name"
+        self.assertEqual(utils.parse_variable(content), "var_name")
+
+    def test_is_functon(self):
+        content = "${func()}"
+        self.assertTrue(utils.is_functon(content))
+        content = "${func(5)}"
+        self.assertTrue(utils.is_functon(content))
+        content = "${func(1, 2)}"
+        self.assertTrue(utils.is_functon(content))
+        content = "${func($a, $b)}"
+        self.assertTrue(utils.is_functon(content))
+        content = "${func(a=1, b=2)}"
+        self.assertTrue(utils.is_functon(content))
+        content = "${func(1, 2, a=3, b=4)}"
+        self.assertTrue(utils.is_functon(content))
+        content = "${func(1, $b, c=$x, d=4)}"
+        self.assertTrue(utils.is_functon(content))
+        content = "${func}"
+        self.assertFalse(utils.is_functon(content))
+        content = "$abc"
+        self.assertFalse(utils.is_functon(content))
+        content = "abc"
+        self.assertFalse(utils.is_functon(content))
+
+    def test_parse_string_value(self):
+        str_value = "123"
+        self.assertEqual(utils.parse_string_value(str_value), 123)
+        str_value = "12.3"
+        self.assertEqual(utils.parse_string_value(str_value), 12.3)
+        str_value = "a123"
+        self.assertEqual(utils.parse_string_value(str_value), "a123")
+        str_value = "$var"
+        self.assertEqual(utils.parse_string_value(str_value), "$var")
+        str_value = "${func}"
+        self.assertEqual(utils.parse_string_value(str_value), "${func}")
+
+    def test_parse_functon(self):
+        content = "${func()}"
+        self.assertEqual(
+            utils.parse_function(content),
+            {'func_name': 'func', 'args': [], 'kwargs': {}}
+        )
+        content = "${func(5)}"
+        self.assertEqual(
+            utils.parse_function(content),
+            {'func_name': 'func', 'args': [5], 'kwargs': {}}
+        )
+        content = "${func(1, 2)}"
+        self.assertEqual(
+            utils.parse_function(content),
+            {'func_name': 'func', 'args': [1, 2], 'kwargs': {}}
+        )
+        content = "${func(a=1, b=2)}"
+        self.assertEqual(
+            utils.parse_function(content),
+            {'func_name': 'func', 'args': [], 'kwargs': {'a': 1, 'b': 2}}
+        )
+        content = "${func(a= 1, b =2)}"
+        self.assertEqual(
+            utils.parse_function(content),
+            {'func_name': 'func', 'args': [], 'kwargs': {'a': 1, 'b': 2}}
+        )
+        content = "${func(1, 2, a=3, b=4)}"
+        self.assertEqual(
+            utils.parse_function(content),
+            {'func_name': 'func', 'args': [1, 2], 'kwargs': {'a': 3, 'b': 4}}
+        )
+
+    def test_query_json(self):
+        json_content = {
+            "ids": [1, 2, 3, 4],
+            "person": {
+                "name": {
+                    "first_name": "Leo",
+                    "last_name": "Lee",
+                },
+                "age": 29,
+                "cities": ["Guangzhou", "Shenzhen"]
+            }
+        }
+        query = "ids.2"
+        result = utils.query_json(json_content, query)
+        self.assertEqual(result, 3)
+
+        query = "ids.str_key"
+        with self.assertRaises(exception.ParamsError):
+            utils.query_json(json_content, query)
+
+        query = "ids.5"
+        with self.assertRaises(exception.ParamsError):
+            utils.query_json(json_content, query)
+
+        query = "person.age"
+        result = utils.query_json(json_content, query)
+        self.assertEqual(result, 29)
+
+        query = "person.not_exist_key"
+        with self.assertRaises(exception.ParamsError):
+            utils.query_json(json_content, query)
+
+        query = "person.cities.0"
+        result = utils.query_json(json_content, query)
+        self.assertEqual(result, "Guangzhou")
+
+        query = "person.name.first_name"
+        result = utils.query_json(json_content, query)
+        self.assertEqual(result, "Leo")
+
+    def test_match_expected(self):
+        self.assertTrue(utils.match_expected(1, 1, "eq"))
+        self.assertTrue(utils.match_expected("abc", "abc", "eq"))
+        self.assertTrue(utils.match_expected("abc", "abc"))
+        self.assertFalse(utils.match_expected(123, "123", "eq"))
+        self.assertFalse(utils.match_expected(123, "123"))
+
+        self.assertTrue(utils.match_expected("123", 3, "len_eq"))
+        self.assertTrue(utils.match_expected(123, "123", "str_eq"))
+        self.assertTrue(utils.match_expected(123, "123", "ne"))
+
+        self.assertTrue(utils.match_expected("123", 2, "len_gt"))
+        self.assertTrue(utils.match_expected("123", 3, "len_ge"))
+        self.assertTrue(utils.match_expected("123", 4, "len_lt"))
+        self.assertTrue(utils.match_expected("123", 3, "len_le"))
+
+        self.assertTrue(utils.match_expected(1, 2, "lt"))
+        self.assertTrue(utils.match_expected(1, 1, "le"))
+        self.assertTrue(utils.match_expected(2, 1, "gt"))
+        self.assertTrue(utils.match_expected(1, 1, "ge"))
+
+        self.assertTrue(utils.match_expected("123abc456", "3ab", "contains"))
+        self.assertTrue(utils.match_expected("3ab", "123abc456", "contained_by"))
+
+        self.assertTrue(utils.match_expected("123abc456", "^123.*456$", "regex"))
+        self.assertFalse(utils.match_expected("123abc456", "^12b.*456$", "regex"))
+
+        with self.assertRaises(exception.ParamsError):
+            utils.match_expected(1, 2, "not_supported_comparator")
+
+        self.assertTrue(utils.match_expected("2017-06-29 17:29:58", 19, "str_len"))
+        self.assertTrue(utils.match_expected("2017-06-29 17:29:58", "19", "str_len"))
+
+        self.assertTrue(utils.match_expected("abc123", "ab", "startswith"))
+        self.assertTrue(utils.match_expected("123abc", 12, "startswith"))
+        self.assertTrue(utils.match_expected(12345, 123, "startswith"))
+
+    def test_deep_update_dict(self):
+        origin_dict = {'a': 1, 'b': {'c': 3, 'd': 4}, 'f': 6}
+        override_dict = {'a': 2, 'b': {'c': 33, 'e': 5}, 'g': 7}
+        updated_dict = utils.deep_update_dict(origin_dict, override_dict)
+        self.assertEqual(
+            updated_dict,
+            {'a': 2, 'b': {'c': 33, 'd': 4, 'e': 5}, 'f': 6, 'g': 7}
+        )