refactor mock server:

1, remove authentication switcher;
2, change authentication method: from verify request data's md5 to get token at the beginning.

README.md

@@ -58,56 +58,57 @@ optional arguments:
 
 如下是一个典型的接口测试用例示例。具体的编写方式请阅读详细文档。
 
-```python
+```yaml
 - config:
     name: "create user testsets."
-    requires:
+    import_module_functions:
-        - random
+        - tests.data.custom_functions
-        - string
-        - hashlib
-    function_binds:
-        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
-        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
     variable_binds:
-        - TOKEN: debugtalk
+        - user_agent: 'iOS/10.3'
-        - data: ""
+        - device_sn: ${gen_random_string(15)}
-        - random: ${gen_random_string(5)}
+        - os_platform: 'ios'
-        - authorization: ${gen_md5($TOKEN, $data, $random)}
+        - app_version: '2.8.6'
+        - sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)}
     request:
         base_url: http://127.0.0.1:5000
+        headers:
+            Content-Type: application/json
+            device_sn: $device_sn
+
+- test:
+    name: get token
+    request:
+        url: /api/get-token
+        method: POST
+        headers:
+            user_agent: $user_agent
+            device_sn: $device_sn
+            os_platform: $os_platform
+            app_version: $app_version
+        json:
+            sign: $sign
+    extract_binds:
+        - token: content.token
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 200}
+        - {"check": "content.token", "comparator": "len_eq", "expected": 16}
 
 - test:
     name: create user which does not exist
     variable_binds:
-        - data: '{"name": "user", "password": "123456"}'
+        - user_name: "user1"
+        - user_password: "123456"
     request:
         url: /api/users/1000
         method: POST
         headers:
-            Content-Type: application/json
+            token: $token
-            authorization: $authorization
+        json:
-            random: $random
+            name: $user_name
-        data: $data
+            password: $user_password
     validators:
         - {"check": "status_code", "comparator": "eq", "expected": 201}
         - {"check": "content.success", "comparator": "eq", "expected": true}
-
-- test:
-    name: create user which does exist
-    variable_binds:
-        - data: '{"name": "user", "password": "123456"}'
-        - expected_status_code: 500
-    request:
-        url: /api/users/1000
-        method: POST
-        headers:
-            Content-Type: application/json
-            authorization: $authorization
-            random: $random
-        data: $data
-    validators:
-        - {"check": "status_code", "comparator": "eq", "expected": 500}
-        - {"check": "content.success", "comparator": "eq", "expected": false}
 ```
 
 ## 运行测试用例

ate/__init__.py

@@ -1 +1 @@
-__version__ = '0.2.0'
+__version__ = '0.2.1'

ate/testcase.py

@@ -1,5 +1,6 @@
 import re
 from ate.exception import ParamsError
+from ate.utils import string_type
 
 
 def parse_content_with_variables(content, variables_binds):
@@ -58,7 +59,7 @@ def parse_template(testcase_template, variables_binds):
     def substitute(content):
         """ substitute content recursively, each variable will be replaced with bind value.
         """
-        if isinstance(content, str):
+        if isinstance(content, string_type):
             return parse_content_with_variables(content, variables_binds)
 
         if isinstance(content, list):

ate/utils.py

@@ -1,12 +1,13 @@
 import ast
 import hashlib
+import hmac
 import json
 import os.path
 import random
 import re
 import string
-import yaml
 
+import yaml
 from ate.exception import ParamsError
 
 try:
@@ -16,6 +17,7 @@ except NameError:
     string_type = str
     PYTHON_VERSION = 3
 
+SECRET_KEY = "DebugTalk"
 variable_regexp = re.compile(r"^\$([\w_]+)$")
 function_regexp = re.compile(r"^\$\{([\w_]+)\(([\$\w_ =,]*)\)\}$")
 
@@ -26,27 +28,11 @@ def gen_random_string(str_len):
 def gen_md5(*str_args):
     return hashlib.md5("".join(str_args).encode('utf-8')).hexdigest()
 
-def handle_req_data(data):
+def get_sign(*args):
-
+    content = ''.join(args).encode('ascii')
-    if PYTHON_VERSION == 3 and isinstance(data, bytes):
+    sign_key = SECRET_KEY.encode('ascii')
-        # In Python3, convert bytes to str
+    sign = hmac.new(sign_key, content, hashlib.sha1).hexdigest()
-        data = data.decode('utf-8')
+    return sign
-
-    if not data:
-        return data
-
-    if isinstance(data, str):
-        # check if data in str can be converted to dict
-        try:
-            data = json.loads(data)
-        except ValueError:
-            pass
-
-    if isinstance(data, dict):
-        # sort data in dict with keys, then convert to str
-        data = json.dumps(data, sort_keys=True)
-
-    return data
 
 def load_yaml_file(yaml_file):
     with open(yaml_file, 'r+') as stream:

tests/api_server.py

@@ -1,9 +1,10 @@
 import hashlib
+import hmac
 import json
 from functools import wraps
 
-from flask import Flask, make_response, request
 from ate import utils
+from flask import Flask, make_response, request
 
 app = Flask(__name__)
 
@@ -22,25 +23,32 @@ data structure:
 """
 users_dict = {}
 
-AUTHENTICATION = False
+""" storage all token data
-TOKEN = "debugtalk"
+data structure:
+    token_dict = {
+        'device_sn1': 'token1',
+        'device_sn2': 'token1'
+    }
+"""
+token_dict = {}
 
 def validate_request(func):
 
     @wraps(func)
-    def wrapper(*args, **kwds):
+    def wrapper(*args, **kwargs):
-        if not AUTHENTICATION:
+        device_sn = request.headers.get('device_sn', "")
-            return func(*args, **kwds)
+        token = request.headers.get('token', "")
 
-        try:
+        if not device_sn or not token:
-            req_headers = request.headers
+            result = {
-            req_authorization = req_headers['Authorization']
+                'success': False,
-            random_str = req_headers['Random']
+                'msg': "device_sn or token is null."
-            data = utils.handle_req_data(request.data)
+            }
-            authorization = utils.gen_md5(TOKEN, data, random_str)
+            response = make_response(json.dumps(result), 401)
-            assert authorization == req_authorization
+            response.headers["Content-Type"] = "application/json"
-            return func(*args, **kwds)
+            return response
-        except (KeyError, AssertionError):
+
+        if token_dict[device_sn] != token:
             result = {
                 'success': False,
                 'msg': "Authorization failed!"
@@ -49,16 +57,46 @@ def validate_request(func):
             response.headers["Content-Type"] = "application/json"
             return response
 
+        return func(*args, **kwargs)
+
     return wrapper
 
 
 @app.route('/')
-@validate_request
 def index():
     return "Hello World!"
 
+@app.route('/api/get-token', methods=['POST'])
+def get_token():
+    user_agent = request.headers.get('User-Agent', "")
+    device_sn = request.headers.get('device_sn', "")
+    os_platform = request.headers.get('os_platform', "")
+    app_version = request.headers.get('app_version', "")
+    data = request.get_json()
+    sign = data.get('sign', "")
+
+    expected_sign = utils.get_sign(user_agent, device_sn, os_platform, app_version)
+
+    if expected_sign != sign:
+        result = {
+            'success': False,
+            'msg': "Authorization failed!"
+        }
+        response = make_response(json.dumps(result), 403)
+    else:
+        token = utils.gen_random_string(16)
+        token_dict[device_sn] = token
+
+        result = {
+            'success': True,
+            'token': token
+        }
+        response = make_response(json.dumps(result))
+
+    response.headers["Content-Type"] = "application/json"
+    return response
+
 @app.route('/customize-response', methods=['POST'])
-@validate_request
 def get_customized_response():
     expected_resp_json = request.get_json()
     status_code = expected_resp_json.get('status_code', 200)
@@ -71,17 +109,6 @@ def get_customized_response():
 
     return response
 
-@app.route('/api/token')
-@validate_request
-def get_token():
-    result = {
-        'success': True,
-        'token': utils.gen_random_string(8)
-    }
-    response = make_response(json.dumps(result))
-    response.headers["Content-Type"] = "application/json"
-    return response
-
 @app.route('/api/users')
 @validate_request
 def get_users():
@@ -95,7 +122,7 @@ def get_users():
     response.headers["Content-Type"] = "application/json"
     return response
 
-@app.route('/api/users', methods=['DELETE'])
+@app.route('/api/reset-all')
 @validate_request
 def clear_users():
     users_dict.clear()

tests/base.py

@@ -2,6 +2,7 @@ import multiprocessing
 import time
 import unittest
 
+import requests
 from ate import utils
 from tests import api_server
 
@@ -10,29 +11,49 @@ class ApiServerUnittest(unittest.TestCase):
     """ Test case class that sets up an HTTP server which can be used within the tests
     """
 
-    authentication = False
-
     @classmethod
     def setUpClass(cls):
-        api_server.AUTHENTICATION = cls.authentication
+        cls.host = "http://127.0.0.1:5000"
         cls.api_server_process = multiprocessing.Process(
             target=api_server.app.run
         )
         cls.api_server_process.start()
         time.sleep(0.1)
+        cls.api_client = requests.Session()
 
     @classmethod
     def tearDownClass(cls):
         cls.api_server_process.terminate()
 
-    def prepare_headers(self, data=""):
+    def get_token(self, user_agent, device_sn, os_platform, app_version):
-        token = api_server.TOKEN
+        url = "%s/api/get-token" % self.host
-        data = utils.handle_req_data(data)
-        random_str = utils.gen_random_string(5)
-        authorization = utils.gen_md5(token, data, random_str)
-
         headers = {
-            'authorization': authorization,
+            'Content-Type': 'application/json',
-            'random': random_str
+            'User-Agent': user_agent,
+            'device_sn': device_sn,
+            'os_platform': os_platform,
+            'app_version': app_version
+        }
+        data = {
+            'sign': utils.get_sign(user_agent, device_sn, os_platform, app_version)
+        }
+
+        resp = self.api_client.post(url, json=data, headers=headers)
+        resp_json = resp.json()
+        self.assertTrue(resp_json["success"])
+        self.assertIn("token", resp_json)
+        self.assertEqual(len(resp_json["token"]), 16)
+        return resp_json["token"]
+
+    def get_authenticated_headers(self):
+        user_agent = 'iOS/10.3'
+        device_sn = utils.gen_random_string(15)
+        os_platform = 'ios'
+        app_version = '2.8.6'
+
+        token = self.get_token(user_agent, device_sn, os_platform, app_version)
+        headers = {
+            'device_sn': device_sn,
+            'token': token
         }
         return headers

tests/data/custom_functions.py

@@ -1,4 +1,5 @@
 import hashlib
+import hmac
 import json
 import random
 import string
@@ -13,37 +14,34 @@ except NameError:
     PYTHON_VERSION = 3
     import urllib.parse as urllib
 
+SECRET_KEY = "DebugTalk"
 
 def gen_random_string(str_len):
-    return ''.join(
+    random_char_list = []
-        random.choice(string.ascii_letters + string.digits) for _ in range(str_len))
+    for _ in range(str_len):
+        random_char = random.choice(string.ascii_letters + string.digits)
+        random_char_list.append(random_char)
+
+    random_string = ''.join(random_char_list)
+    return random_string
+
+gen_random_string_lambda = lambda str_len: ''.join(
+    random.choice(string.ascii_letters + string.digits) for _ in range(str_len))
+
+def get_sign(*args):
+    content = ''.join(args).encode('ascii')
+    sign_key = SECRET_KEY.encode('ascii')
+    sign = hmac.new(sign_key, content, hashlib.sha1).hexdigest()
+    return sign
+
+get_sign_lambda = lambda *args: hmac.new(
+    'DebugTalk'.encode('ascii'),
+    ''.join(args).encode('ascii'),
+    hashlib.sha1).hexdigest()
 
 def gen_md5(*args):
-    args = [handle_req_data(item) for item in args]
     return hashlib.md5("".join(args).encode('utf-8')).hexdigest()
 
-def handle_req_data(data):
-
-    if PYTHON_VERSION == 3 and isinstance(data, bytes):
-        # In Python3, convert bytes to str
-        data = data.decode('utf-8')
-
-    if not data:
-        return data
-
-    if isinstance(data, str):
-        # check if data in str can be converted to dict
-        try:
-            data = json.loads(data)
-        except ValueError:
-            pass
-
-    if isinstance(data, dict):
-        # sort data in dict with keys, then convert to str
-        data = json.dumps(data, sort_keys=True)
-
-    return data
-
 def gen_urlencode_str(**kargs):
     urlencoded_str = ""
     quote_times = int(kargs.pop("quote_times", 1))

tests/data/demo_import_functions.yml

@@ -1,42 +0,0 @@
-- config:
-    name: "create user testsets."
-    import_module_functions:
-        - tests.data.custom_functions
-    variable_binds:
-        - TOKEN: debugtalk
-
-- test:
-    name: create user which does not exist
-    variable_binds:
-        - json: {"name": "user", "password": "123456"}
-        - random: ${gen_random_string(5)}
-        - authorization: ${gen_md5($TOKEN, $json, $random)}
-    request:
-        url: http://127.0.0.1:5000/api/users/1000
-        method: POST
-        headers:
-            Content-Type: application/json
-            authorization: $authorization
-            random: $random
-        json: $json
-    validators:
-        - {"check": "status_code", "comparator": "eq", "expected": 201}
-        - {"check": "content.success", "comparator": "eq", "expected": true}
-
-- test:
-    name: create user which does not exist
-    variable_binds:
-        - json: {"name": "user", "password": "123456"}
-        - random: ${gen_random_string(5)}
-        - authorization: ${gen_md5($TOKEN, $json, $random)}
-    request:
-        url: http://127.0.0.1:5000/api/users/1000
-        method: POST
-        headers:
-            Content-Type: application/json
-            authorization: $authorization
-            random: $random
-        json: $json
-    validators:
-        - {"check": "status_code", "comparator": "eq", "expected": 500}
-        - {"check": "content.success", "comparator": "eq", "expected": false}

tests/data/demo_template_separate.yml

@@ -1,52 +0,0 @@
-
-- test:
-    name: create user which does not exist
-    requires:
-        - random
-        - string
-        - hashlib
-    function_binds:
-        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
-        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
-    variable_binds:
-        - TOKEN: debugtalk
-        - random: ${gen_random_string(5)}
-        - data: '{"name": "user", "password": "123456"}'
-        - authorization: ${gen_md5($TOKEN, $data, $random)}
-    request:
-        url: http://127.0.0.1:5000/api/users/1000
-        method: POST
-        headers:
-            Content-Type: application/json
-            authorization: $authorization
-            random: $random
-        data: $data
-    validators:
-        - {"check": "status_code", "comparator": "eq", "expected": 201}
-        - {"check": "content.success", "comparator": "eq", "expected": true}
-
-- test:
-    name: create user which does exist
-    requires:
-        - random
-        - string
-        - hashlib
-    function_binds:
-        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
-        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
-    variable_binds:
-        - TOKEN: debugtalk
-        - random: ${gen_random_string(5)}
-        - data: '{"name": "user", "password": "123456"}'
-        - authorization: ${gen_md5($TOKEN, $data, $random)}
-    request:
-        url: http://127.0.0.1:5000/api/users/1000
-        method: POST
-        headers:
-            Content-Type: application/json
-            authorization: $authorization
-            random: $random
-        data: $data
-    validators:
-        - {"check": "status_code", "comparator": "eq", "expected": 500}
-        - {"check": "content.success", "comparator": "eq", "expected": false}

tests/data/demo_template_sets.yml

@@ -1,50 +0,0 @@
-- config:
-    name: "create user testsets."
-    requires:
-        - random
-        - string
-        - hashlib
-    function_binds:
-        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
-        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
-    variable_binds:
-        - TOKEN: debugtalk
-    request:
-        base_url: http://127.0.0.1:5000
-
-- test:
-    name: create user which does not exist
-    variable_binds:
-        - data: '{"name": "user", "password": "123456"}'
-        - random: ${gen_random_string(5)}
-        - authorization: ${gen_md5($TOKEN, $data, $random)}
-    request:
-        url: /api/users/1000
-        method: POST
-        headers:
-            Content-Type: application/json
-            authorization: $authorization
-            random: $random
-        data: $data
-    validators:
-        - {"check": "status_code", "comparator": "eq", "expected": 201}
-        - {"check": "content.success", "comparator": "eq", "expected": true}
-
-- test:
-    name: create user which does exist
-    variable_binds:
-        - data: '{"name": "user", "password": "123456"}'
-        - random: ${gen_random_string(5)}
-        - authorization: ${gen_md5($TOKEN, $data, $random)}
-        - expected_status_code: 500
-    request:
-        url: /api/users/1000
-        method: POST
-        headers:
-            Content-Type: application/json
-            authorization: $authorization
-            random: $random
-        data: $data
-    validators:
-        - {"check": "status_code", "comparator": "eq", "expected": 500}
-        - {"check": "content.success", "comparator": "eq", "expected": false}

tests/data/demo_testset_hardcode.json

@@ -0,0 +1,74 @@
+[
+    {
+        "test": {
+            "name": "get token",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/get-token",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json",
+                    "user_agent": "iOS/10.3",
+                    "device_sn": "HZfFBh6tU59EdXJ",
+                    "os_platform": "ios",
+                    "app_version": "2.8.6"
+                },
+                "json": {
+                    "sign": "f1219719911caae89ccc301679857ebfda115ca2"
+                }
+            },
+            "extract_binds": [
+                {
+                    "token": "content.token"
+                }
+            ],
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 200},
+                {"check": "content.token", "comparator": "len_eq", "expected": 16}
+            ]
+        }
+    },
+    {
+        "test": {
+            "name": "create user which does not exist",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/2000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json",
+                    "device_sn": "HZfFBh6tU59EdXJ",
+                    "token": "$token"
+                },
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
+            },
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 201},
+                {"check": "content.success", "comparator": "eq", "expected": true}
+            ]
+        }
+    },
+    {
+        "test": {
+            "name": "create user which existed",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/2000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json",
+                    "device_sn": "HZfFBh6tU59EdXJ",
+                    "token": "$token"
+                },
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
+            },
+            "validators": [
+                {"check": "status_code", "comparator": "eq", "expected": 500},
+                {"check": "content.success", "comparator": "eq", "expected": false}
+            ]
+        }
+    }
+]

tests/data/demo_testset_hardcode.yml

@@ -1,3 +1,22 @@
+- test:
+    name: get token
+    request:
+        url: http://127.0.0.1:5000/api/get-token
+        method: POST
+        headers:
+            Content-Type: application/json
+            user_agent: 'iOS/10.3'
+            device_sn: 'HZfFBh6tU59EdXJ'
+            os_platform: 'ios'
+            app_version: '2.8.6'
+        json:
+            sign: f1219719911caae89ccc301679857ebfda115ca2
+    extract_binds:
+        - token: content.token
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 200}
+        - {"check": "content.token", "comparator": "len_eq", "expected": 16}
+
 - test:
     name: create user which does not exist
     request:
@@ -5,8 +24,8 @@
         method: POST
         headers:
             Content-Type: application/json
-            authorization: a83de0ff8d2e896dbd8efb81ba14e17d
+            device_sn: 'HZfFBh6tU59EdXJ'
-            random: A2dEx
+            token: $token
         json:
             name: "user1"
             password: "123456"
@@ -21,8 +40,8 @@
         method: POST
         headers:
             Content-Type: application/json
-            authorization: a83de0ff8d2e896dbd8efb81ba14e17d
+            device_sn: 'HZfFBh6tU59EdXJ'
-            random: A2dEx
+            token: $token
         json:
             name: "user1"
             password: "123456"

tests/data/demo_testset_template_import_functions.yml

@@ -0,0 +1,64 @@
+- config:
+    name: "create user testsets."
+    import_module_functions:
+        - tests.data.custom_functions
+    variable_binds:
+        - user_agent: 'iOS/10.3'
+        - device_sn: ${gen_random_string(15)}
+        - os_platform: 'ios'
+        - app_version: '2.8.6'
+        - sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)}
+    request:
+        base_url: http://127.0.0.1:5000
+        headers:
+            Content-Type: application/json
+            device_sn: $device_sn
+
+- test:
+    name: get token
+    request:
+        url: /api/get-token
+        method: POST
+        headers:
+            user_agent: $user_agent
+            device_sn: $device_sn
+            os_platform: $os_platform
+            app_version: $app_version
+        json:
+            sign: $sign
+    extract_binds:
+        - token: content.token
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 200}
+        - {"check": "content.token", "comparator": "len_eq", "expected": 16}
+
+- test:
+    name: create user which does not exist
+    variable_binds:
+        - user_name: "user1"
+        - user_password: "123456"
+    request:
+        url: /api/users/1000
+        method: POST
+        headers:
+            token: $token
+        json:
+            name: $user_name
+            password: $user_password
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
+
+- test:
+    name: create user which does not exist
+    request:
+        url: /api/users/1000
+        method: POST
+        headers:
+            token: $token
+        json:
+            name: "user1"
+            password: "123456"
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

tests/data/demo_testset_template_lambda_functions.yml

@@ -0,0 +1,74 @@
+- config:
+    name: "create user testsets."
+    requires:
+        - random
+        - string
+        - hashlib
+        - hmac
+    function_binds:
+        gen_random_string_lambda: "lambda str_len: ''.join(
+            random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
+        get_sign_lambda: "lambda *args: hmac.new(
+            'DebugTalk'.encode('ascii'),
+            ''.join(args).encode('ascii'),
+            hashlib.sha1).hexdigest()"
+    variable_binds:
+        - user_agent: 'iOS/10.3'
+        - device_sn: ${gen_random_string_lambda(15)}
+        - os_platform: 'ios'
+        - app_version: '2.8.6'
+        - sign: ${get_sign_lambda($user_agent, $device_sn, $os_platform, $app_version)}
+    request:
+        base_url: http://127.0.0.1:5000
+        headers:
+            Content-Type: application/json
+            device_sn: $device_sn
+
+- test:
+    name: get token
+    request:
+        url: /api/get-token
+        method: POST
+        headers:
+            user_agent: $user_agent
+            device_sn: $device_sn
+            os_platform: $os_platform
+            app_version: $app_version
+        json:
+            sign: $sign
+    extract_binds:
+        - token: content.token
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 200}
+        - {"check": "content.token", "comparator": "len_eq", "expected": 16}
+
+- test:
+    name: create user which does not exist
+    variable_binds:
+        - user_name: "user1"
+        - user_password: "123456"
+    request:
+        url: /api/users/1000
+        method: POST
+        headers:
+            token: $token
+        json:
+            name: $user_name
+            password: $user_password
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
+
+- test:
+    name: create user which does not exist
+    request:
+        url: /api/users/1000
+        method: POST
+        headers:
+            token: $token
+        json:
+            name: "user1"
+            password: "123456"
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

tests/data/demo_testset_variables.yml

@@ -0,0 +1,62 @@
+- config:
+    name: "create user testsets."
+    variable_binds:
+        - user_agent: 'iOS/10.3'
+        - device_sn: 'HZfFBh6tU59EdXJ'
+        - os_platform: 'ios'
+        - app_version: '2.8.6'
+    request:
+        base_url: http://127.0.0.1:5000
+        headers:
+            Content-Type: application/json
+            device_sn: $device_sn
+
+- test:
+    name: get token
+    request:
+        url: /api/get-token
+        method: POST
+        headers:
+            Content-Type: application/json
+            user_agent: $user_agent
+            device_sn: $device_sn
+            os_platform: $os_platform
+            app_version: $app_version
+        json:
+            sign: f1219719911caae89ccc301679857ebfda115ca2
+    extract_binds:
+        - token: content.token
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 200}
+        - {"check": "content.token", "comparator": "len_eq", "expected": 16}
+
+- test:
+    name: create user which does not exist
+    variable_binds:
+        - user_name: "user1"
+        - user_password: "123456"
+    request:
+        url: /api/users/1000
+        method: POST
+        headers:
+            token: $token
+        json:
+            name: $user_name
+            password: $user_password
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 201}
+        - {"check": "content.success", "comparator": "eq", "expected": true}
+
+- test:
+    name: create user which does not exist
+    request:
+        url: /api/users/1000
+        method: POST
+        headers:
+            token: $token
+        json:
+            name: "user1"
+            password: "123456"
+    validators:
+        - {"check": "status_code", "comparator": "eq", "expected": 500}
+        - {"check": "content.success", "comparator": "eq", "expected": false}

tests/data/simple_demo_auth_hardcode.json

@@ -1,46 +0,0 @@
-[
-    {
-        "test": {
-            "name": "create user which does not exist",
-            "request": {
-                "url": "http://127.0.0.1:5000/api/users/1000",
-                "method": "POST",
-                "headers": {
-                    "content-type": "application/json",
-                    "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
-                    "random": "A2dEx"
-                },
-                "json": {
-                    "name": "user1",
-                    "password": "123456"
-                }
-            },
-            "validators": [
-                {"check": "status_code", "comparator": "eq", "expected": 201},
-                {"check": "content.success", "comparator": "eq", "expected": true}
-            ]
-        }
-    },
-    {
-        "test": {
-            "name": "create user which existed",
-            "request": {
-                "url": "http://127.0.0.1:5000/api/users/1000",
-                "method": "POST",
-                "headers": {
-                    "content-type": "application/json",
-                    "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
-                    "random": "A2dEx"
-                },
-                "json": {
-                    "name": "user1",
-                    "password": "123456"
-                }
-            },
-            "validators": [
-                {"check": "status_code", "comparator": "eq", "expected": 500},
-                {"check": "content.success", "comparator": "eq", "expected": false}
-            ]
-        }
-    }
-]

tests/data/simple_demo_no_auth.json

@@ -1,43 +0,0 @@
-[
-    {
-        "test": {
-            "name": "create user which does not exist",
-            "request": {
-                "url": "http://127.0.0.1:5000/api/users/1000",
-                "method": "POST",
-                "headers": {
-                    "content-type": "application/json"
-                },
-                "cookies": {},
-                "json": {
-                    "name": "user1",
-                    "password": "123456"
-                }
-            },
-            "validators": [
-                {"check": "status_code", "comparator": "eq", "expected": 201},
-                {"check": "content.success", "comparator": "eq", "expected": true}
-            ]
-        }
-    },
-    {
-        "test": {
-            "name": "create user which existed",
-            "request": {
-                "url": "http://127.0.0.1:5000/api/users/1000",
-                "method": "POST",
-                "headers": {
-                    "content-type": "application/json"
-                },
-                "json": {
-                    "name": "user1",
-                    "password": "123456"
-                }
-            },
-            "validators": [
-                {"check": "status_code", "comparator": "eq", "expected": 500},
-                {"check": "content.success", "comparator": "eq", "expected": false}
-            ]
-        }
-    }
-]

tests/data/simple_demo_no_auth.yml

@@ -1,29 +0,0 @@
-- test:
-    name: create user which does not exist
-    request:
-        url: http://127.0.0.1:5000/api/users/1000
-        method: POST
-        headers:
-            Content-Type: application/json
-        json:
-            name: user1
-            password: 123456
-    validators:
-        - {"check": "status_code", "comparator": "eq", "expected": 201}
-        - {"check": "content.success", "comparator": "eq", "expected": true}
-        - {"check": "headers.content-type", "comparator": "eq", "expected": "application/json"}
-
-- test:
-    name: create user which existed
-    request:
-        url: http://127.0.0.1:5000/api/users/1000
-        method: POST
-        headers:
-            Content-Type: application/json
-        json:
-            name: user1
-            password: 123456
-    validators:
-        - {"check": "status_code", "comparator": "eq", "expected": 500}
-        - {"check": "content.success", "comparator": "eq", "expected": false}
-        - {"check": "headers.content-type", "comparator": "eq", "expected": "application/json"}

tests/test_apiserver.py

@@ -1,24 +1,30 @@
-import requests
 import random
+import requests
+
 from tests.base import ApiServerUnittest
 
+
 class TestApiServer(ApiServerUnittest):
+
     def setUp(self):
         super(TestApiServer, self).setUp()
-        self.host = "http://127.0.0.1:5000"
+        self.headers = self.get_authenticated_headers()
-        self.api_client = requests.Session()
+        self.reset_all()
-        self.clear_users()
 
     def tearDown(self):
         super(TestApiServer, self).tearDown()
 
-    def clear_users(self):
+    def test_index(self):
-        url = "%s/api/users" % self.host
+        resp = self.api_client.get(self.host)
-        return self.api_client.delete(url)
+        self.assertEqual(200, resp.status_code)
+
+    def reset_all(self):
+        url = "%s/api/reset-all" % self.host
+        return self.api_client.get(url, headers=self.headers)
 
     def get_users(self):
         url = "%s/api/users" % self.host
-        return self.api_client.get(url)
+        return self.api_client.get(url, headers=self.headers)
 
     def create_user(self, uid, name, password):
         url = "%s/api/users/%d" % (self.host, uid)
@@ -26,11 +32,11 @@ class TestApiServer(ApiServerUnittest):
             'name': name,
             'password': password
         }
-        return self.api_client.post(url, json=data)
+        return self.api_client.post(url, headers=self.headers, json=data)
 
     def get_user(self, uid):
         url = "%s/api/users/%d" % (self.host, uid)
-        return self.api_client.get(url)
+        return self.api_client.get(url, headers=self.headers)
 
     def update_user(self, uid, name, password):
         url = "%s/api/users/%d" % (self.host, uid)
@@ -38,14 +44,14 @@ class TestApiServer(ApiServerUnittest):
             'name': name,
             'password': password
         }
-        return self.api_client.put(url, json=data)
+        return self.api_client.put(url, headers=self.headers, json=data)
 
     def delete_user(self, uid):
         url = "%s/api/users/%d" % (self.host, uid)
-        return self.api_client.delete(url)
+        return self.api_client.delete(url, headers=self.headers)
 
-    def test_clear_users(self):
+    def test_reset_all(self):
-        resp = self.clear_users()
+        resp = self.reset_all()
         self.assertEqual(200, resp.status_code)
         self.assertEqual(True, resp.json()['success'])
 
@@ -114,7 +120,11 @@ class TestApiServer(ApiServerUnittest):
         expected_response = {
             'status_code': status_code,
         }
-        resp = self.api_client.post(url, json=expected_response)
+        resp = self.api_client.post(
+            url,
+            headers=self.headers,
+            json=expected_response
+        )
         self.assertEqual(status_code, resp.status_code)
 
     def test_get_customized_response_headers(self):
@@ -125,13 +135,10 @@ class TestApiServer(ApiServerUnittest):
             }
         }
         url = "%s/customize-response" % self.host
-        resp = self.api_client.post(url, json=expected_response)
+        resp = self.api_client.post(
+            url,
+            headers=self.headers,
+            json=expected_response
+        )
         self.assertIn('abc', resp.headers)
         self.assertIn('123', resp.headers['abc'])
-
-    def test_get_token(self):
-        url = "%s/api/token" % self.host
-        resp = self.api_client.get(url)
-        resp_json = resp.json()
-        self.assertTrue(resp_json["success"])
-        self.assertEqual(len(resp_json["token"]), 8)

tests/test_apiserver_v2.py

@@ -1,158 +0,0 @@
-import random
-import requests
-
-from tests.base import ApiServerUnittest
-
-
-class TestApiServerV2(ApiServerUnittest):
-
-    authentication = True
-
-    def setUp(self):
-        super(TestApiServerV2, self).setUp()
-        self.host = "http://127.0.0.1:5000"
-        self.api_client = requests.Session()
-        self.clear_users()
-
-    def tearDown(self):
-        super(TestApiServerV2, self).tearDown()
-
-    def test_index(self):
-        headers = self.prepare_headers()
-        resp = self.api_client.get(self.host, headers=headers)
-        self.assertEqual(200, resp.status_code)
-
-    def clear_users(self):
-        url = "%s/api/users" % self.host
-        return self.api_client.delete(url, headers=self.prepare_headers())
-
-    def get_users(self):
-        url = "%s/api/users" % self.host
-        return self.api_client.get(url, headers=self.prepare_headers())
-
-    def create_user(self, uid, name, password):
-        url = "%s/api/users/%d" % (self.host, uid)
-        data = {
-            'name': name,
-            'password': password
-        }
-        headers = self.prepare_headers(data)
-        return self.api_client.post(url, headers=headers, json=data)
-
-    def get_user(self, uid):
-        url = "%s/api/users/%d" % (self.host, uid)
-        return self.api_client.get(url, headers=self.prepare_headers())
-
-    def update_user(self, uid, name, password):
-        url = "%s/api/users/%d" % (self.host, uid)
-        data = {
-            'name': name,
-            'password': password
-        }
-        headers = self.prepare_headers(data)
-        return self.api_client.put(url, headers=headers, json=data)
-
-    def delete_user(self, uid):
-        url = "%s/api/users/%d" % (self.host, uid)
-        return self.api_client.delete(url, headers=self.prepare_headers())
-
-    def test_clear_users(self):
-        resp = self.clear_users()
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual(True, resp.json()['success'])
-
-    def test_create_user_not_existed(self):
-        resp = self.create_user(1000, 'user1', '123456')
-        self.assertEqual(201, resp.status_code)
-        self.assertEqual(True, resp.json()['success'])
-
-    def test_create_user_existed(self):
-        resp = self.create_user(1000, 'user1', '123456')
-        resp = self.create_user(1000, 'user1', '123456')
-        self.assertEqual(500, resp.status_code)
-
-    def test_get_users_empty(self):
-        resp = self.get_users()
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual(resp.json()['count'], 0)
-
-    def test_get_users_not_empty(self):
-        resp = self.create_user(1000, 'user1', '123456')
-        resp = self.get_users()
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual(resp.json()['count'], 1)
-
-        resp = self.create_user(1001, 'user2', '123456')
-        resp = self.get_users()
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual(resp.json()['count'], 2)
-
-    def test_get_user_not_existed(self):
-        resp = self.get_user(1000)
-        self.assertEqual(404, resp.status_code)
-        self.assertEqual(resp.json()['success'], False)
-
-    def test_get_user_existed(self):
-        self.create_user(1000, 'user1', '123456')
-        resp = self.get_user(1000)
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual(resp.json()['success'], True)
-
-    def test_update_user_not_existed(self):
-        resp = self.update_user(1000, 'user1', '123456')
-        self.assertEqual(404, resp.status_code)
-        self.assertEqual(resp.json()['success'], False)
-
-    def test_update_user_existed(self):
-        self.create_user(1000, 'user1', '123456')
-        resp = self.update_user(1000, 'user2', '123456')
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual(resp.json()['data']['name'], 'user2')
-
-    def test_delete_user_not_existed(self):
-        resp = self.delete_user(1000)
-        self.assertEqual(404, resp.status_code)
-        self.assertEqual(resp.json()['success'], False)
-
-    def test_delete_user_existed(self):
-        self.create_user(1000, 'leo', '123456')
-        resp = self.delete_user(1000)
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual(resp.json()['success'], True)
-
-    def test_get_customized_response_status_code(self):
-        status_code = random.randint(200, 511)
-        url = "%s/customize-response" % self.host
-        expected_response = {
-            'status_code': status_code,
-        }
-        resp = self.api_client.post(
-            url,
-            headers=self.prepare_headers(expected_response),
-            json=expected_response
-        )
-        self.assertEqual(status_code, resp.status_code)
-
-    def test_get_customized_response_headers(self):
-        expected_response = {
-            'headers': {
-                'abc': 123,
-                'def': 456
-            }
-        }
-        url = "%s/customize-response" % self.host
-        resp = self.api_client.post(
-            url,
-            headers=self.prepare_headers(expected_response),
-            json=expected_response
-        )
-        self.assertIn('abc', resp.headers)
-        self.assertIn('123', resp.headers['abc'])
-
-    def test_get_token(self):
-        url = "%s/api/token" % self.host
-        headers = self.prepare_headers()
-        resp = self.api_client.get(url, headers=headers)
-        resp_json = resp.json()
-        self.assertTrue(resp_json["success"])
-        self.assertEqual(len(resp_json["token"]), 8)

tests/test_client.py

@@ -4,16 +4,17 @@ from tests.base import ApiServerUnittest
 class TestHttpClient(ApiServerUnittest):
     def setUp(self):
         super(TestHttpClient, self).setUp()
-        self.host = "http://127.0.0.1:5000"
         self.api_client = HttpSession(self.host)
-        self.clear_users()
+        self.headers = self.get_authenticated_headers()
+        self.reset_all()
 
     def tearDown(self):
         super(TestHttpClient, self).tearDown()
 
-    def clear_users(self):
+    def reset_all(self):
-        url = "%s/api/users" % self.host
+        url = "%s/api/reset-all" % self.host
-        return self.api_client.delete(url)
+        headers = self.get_authenticated_headers()
+        return self.api_client.get(url, headers=headers)
 
     def test_request_with_full_url(self):
         url = "%s/api/users/1000" % self.host
@@ -21,7 +22,7 @@ class TestHttpClient(ApiServerUnittest):
             'name': 'user1',
             'password': '123456'
         }
-        resp = self.api_client.post(url, json=data)
+        resp = self.api_client.post(url, json=data, headers=self.headers)
         self.assertEqual(201, resp.status_code)
         self.assertEqual(True, resp.json()['success'])
 
@@ -31,6 +32,6 @@ class TestHttpClient(ApiServerUnittest):
             'name': 'user1',
             'password': '123456'
         }
-        resp = self.api_client.post(url, json=data)
+        resp = self.api_client.post(url, json=data, headers=self.headers)
         self.assertEqual(201, resp.status_code)
         self.assertEqual(True, resp.json()['success'])

tests/test_runner.py

@@ -7,49 +7,58 @@ class TestRunner(ApiServerUnittest):
 
     def setUp(self):
         self.test_runner = runner.Runner()
-        self.clear_users()
+        self.reset_all()
 
-    def clear_users(self):
+        self.testcase_file_path_list = [
-        url = "http://127.0.0.1:5000/api/users"
+            os.path.join(
-        return requests.delete(url)
+                os.getcwd(), 'tests/data/demo_testset_hardcode.yml'),
+            os.path.join(
+                os.getcwd(), 'tests/data/demo_testset_hardcode.json')
+        ]
 
-    def test_run_single_testcase_yaml_success(self):
+    def reset_all(self):
-        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+        url = "%s/api/reset-all" % self.host
-        testcases = utils.load_testcases(testcase_file_path)
+        headers = self.get_authenticated_headers()
-        testcase = testcases[0]["test"]
+        return self.api_client.get(url, headers=headers)
-        success, _ = self.test_runner.run_test(testcase)
-        self.assertTrue(success)
 
-    def test_run_single_testcase_json_success(self):
+    def test_run_single_testcase(self):
-        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.json')
+        for testcase_file_path in self.testcase_file_path_list:
-        testcases = utils.load_testcases(testcase_file_path)
+            testcases = utils.load_testcases(testcase_file_path)
-        testcase = testcases[0]["test"]
+            testcase = testcases[0]["test"]
-        success, _ = self.test_runner.run_test(testcase)
+            success, _ = self.test_runner.run_test(testcase)
-        self.assertTrue(success)
+            self.assertTrue(success)
+
+            testcase = testcases[1]["test"]
+            success, _ = self.test_runner.run_test(testcase)
+            self.assertTrue(success)
+
+            testcase = testcases[2]["test"]
+            success, _ = self.test_runner.run_test(testcase)
+            self.assertTrue(success)
 
     def test_run_single_testcase_fail(self):
         testcase = {
-            "name": "create user which does not exist",
+            "name": "get token",
             "request": {
-                "url": "http://127.0.0.1:5000/api/users/1000",
+                "url": "http://127.0.0.1:5000/api/get-token",
                 "method": "POST",
                 "headers": {
-                    "content-type": "application/json"
+                    "content-type": "application/json",
+                    "user_agent": "iOS/10.3",
+                    "device_sn": "HZfFBh6tU59EdXJ",
+                    "os_platform": "ios",
+                    "app_version": "2.8.6"
                 },
                 "json": {
-                    "name": "user1",
+                    "sign": "f1219719911caae89ccc301679857ebfda115ca2"
-                    "password": "123456"
                 }
             },
             "extract_binds": [
-                {"resp_status_code": "status_code"},
+                {"token": "content.token"}
-                {"resp_body_success": "content.success"},
-                {"resp_headers_contenttype": "headers.content-type"}
             ],
             "validators": [
-                {"check": "resp_status_code", "comparator": "eq", "expected": 200},
+                {"check": "status_code", "comparator": "eq", "expected": 205},
-                {"check": "resp_body_success", "comparator": "eq", "expected": False},
+                {"check": "content.token", "comparator": "len_eq", "expected": 19}
-                {"check": "resp_headers_contenttype", "comparator": "eq", "expected": "html/text"}
             ]
         }
 
@@ -57,41 +66,51 @@ class TestRunner(ApiServerUnittest):
         self.assertFalse(success)
         self.assertEqual(
             diff_content_list[0],
-            {"check": "resp_status_code", "comparator": "eq", "expected": 200, 'value': 201}
+            {"check": "status_code", "comparator": "eq", "expected": 205, 'value': 200}
-        )
-        self.assertEqual(
-            diff_content_list[1],
-            {"check": "resp_body_success", "comparator": "eq", "expected": False, 'value': True}
-        )
-        self.assertEqual(
-            diff_content_list[2],
-            {"check": "resp_headers_contenttype", "comparator": "eq", "expected": "html/text", 'value': "application/json"}
         )
 
-    def test_run_testset_json_success(self):
+    def test_run_testset_hardcode(self):
-        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.json')
+        for testcase_file_path in self.testcase_file_path_list:
+            testsets = utils.load_testcases_by_path(testcase_file_path)
+            results = self.test_runner.run_testset(testsets[0])
+            self.assertEqual(len(results), 3)
+            self.assertEqual(results, [(True, [])] * 3)
+
+    def test_run_testsets_hardcode(self):
+        for testcase_file_path in self.testcase_file_path_list:
+            testsets = utils.load_testcases_by_path(testcase_file_path)
+            results = self.test_runner.run_testsets(testsets)
+            self.assertEqual(len(results), 1)
+            self.assertEqual(results, [[(True, [])] * 3])
+
+    def test_run_testset_template_variables(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_testset_variables.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testset(testsets[0])
-        self.assertEqual(len(results), 2)
+        self.assertEqual(len(results), 3)
-        self.assertEqual(results, [(True, []), (True, [])])
+        self.assertEqual(results, [(True, [])] * 3)
 
-    def test_run_testsets_json_success(self):
+    def test_run_testset_template_import_functions(self):
-        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.json')
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_testset_template_import_functions.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
+        self.assertEqual(len(results), 3)
+        self.assertEqual(results, [(True, [])] * 3)
+
+    def test_run_testsets_template_import_functions(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'tests/data/demo_testset_template_import_functions.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, []), (True, [])])
+        self.assertEqual(results, [[(True, [])] * 3])
 
-    def test_run_testset_yaml_success(self):
+    def test_run_testsets_template_lambda_functions(self):
-        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+        testcase_file_path = os.path.join(
-        testsets = utils.load_testcases_by_path(testcase_file_path)
+            os.getcwd(), 'tests/data/demo_testset_template_lambda_functions.yml')
-        results = self.test_runner.run_testset(testsets[0])
-        self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, []), (True, [])])
-
-    def test_run_testsets_yaml_success(self):
-        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
         self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, []), (True, [])])
+        self.assertEqual(results, [[(True, [])] * 3])

tests/test_runner_v2.py

@@ -1,105 +0,0 @@
-import os
-import requests
-from ate import runner, exception, utils
-from tests.base import ApiServerUnittest
-
-class TestRunnerV2(ApiServerUnittest):
-
-    authentication = True
-
-    def setUp(self):
-        self.test_runner = runner.Runner()
-        self.clear_users()
-
-    def clear_users(self):
-        url = "http://127.0.0.1:5000/api/users"
-        return requests.delete(url, headers=self.prepare_headers())
-
-    def test_run_single_testcase_yaml(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.yml')
-        testcases = utils.load_testcases(testcase_file_path)
-        testcase = testcases[0]["test"]
-        success, _ = self.test_runner.run_test(testcase)
-        self.assertTrue(success)
-
-    def test_run_single_testcase_json(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.json')
-        testcases = utils.load_testcases(testcase_file_path)
-        testcase = testcases[0]["test"]
-        success, _ = self.test_runner.run_test(testcase)
-        self.assertTrue(success)
-
-    def test_run_testset_auth_yaml(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.yml')
-        testsets = utils.load_testcases_by_path(testcase_file_path)
-        results = self.test_runner.run_testset(testsets[0])
-        self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, []), (True, [])])
-
-    def test_run_testsets_auth_yaml(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.yml')
-        testsets = utils.load_testcases_by_path(testcase_file_path)
-        results = self.test_runner.run_testsets(testsets)
-        self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, []), (True, [])])
-
-    def test_run_testset_auth_json(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.json')
-        testsets = utils.load_testcases_by_path(testcase_file_path)
-        results = self.test_runner.run_testset(testsets[0])
-        self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, []), (True, [])])
-
-    def test_run_testsets_auth_json(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/simple_demo_auth_hardcode.json')
-        testsets = utils.load_testcases_by_path(testcase_file_path)
-        results = self.test_runner.run_testsets(testsets)
-        self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, []), (True, [])])
-
-    def test_run_testcase_template_yaml(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/demo_template_separate.yml')
-        testcases = utils.load_testcases(testcase_file_path)
-        success, _ = self.test_runner.run_test(testcases[0]["test"])
-        self.assertTrue(success)
-        success, _ = self.test_runner.run_test(testcases[1]["test"])
-        self.assertTrue(success)
-
-    def test_run_testset_template_yaml(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/demo_template_sets.yml')
-        testsets = utils.load_testcases_by_path(testcase_file_path)
-        results = self.test_runner.run_testset(testsets[0])
-        self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, []), (True, [])])
-
-    def test_run_testsets_template_yaml(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/demo_template_sets.yml')
-        testsets = utils.load_testcases_by_path(testcase_file_path)
-        results = self.test_runner.run_testsets(testsets)
-        self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, []), (True, [])])
-
-    def test_run_testset_template_import_functions(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/demo_import_functions.yml')
-        testsets = utils.load_testcases_by_path(testcase_file_path)
-        results = self.test_runner.run_testset(testsets[0])
-        self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, []), (True, [])])
-
-    def test_run_testsets_template_import_functions(self):
-        testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/demo_import_functions.yml')
-        testsets = utils.load_testcases_by_path(testcase_file_path)
-        results = self.test_runner.run_testsets(testsets)
-        self.assertEqual(len(results), 1)
-        self.assertEqual(results[0], [(True, []), (True, [])])

tests/test_task.py

@@ -1,30 +1,29 @@
 import os
-import random
-import requests
 from tests.base import ApiServerUnittest
 from ate import task, utils
 
-class TestMain(ApiServerUnittest):
+class TestTask(ApiServerUnittest):
 
     def setUp(self):
-        self.clear_users()
+        self.reset_all()
 
-    def clear_users(self):
+    def reset_all(self):
-        url = "http://127.0.0.1:5000/api/users"
+        url = "%s/api/reset-all" % self.host
-        return requests.delete(url)
+        headers = self.get_authenticated_headers()
+        return self.api_client.get(url, headers=headers)
 
     def test_create_suite(self):
-        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_testset_variables.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         suite = task.create_suite(testsets[0])
-        self.assertEqual(suite.countTestCases(), 2)
+        self.assertEqual(suite.countTestCases(), 3)
         for testcase in suite:
             self.assertIsInstance(testcase, task.ApiTestCase)
 
     def test_create_task(self):
-        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+        testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_testset_variables.yml')
         task_suite = task.create_task(testcase_file_path)
-        self.assertEqual(task_suite.countTestCases(), 2)
+        self.assertEqual(task_suite.countTestCases(), 3)
         for suite in task_suite:
             for testcase in suite:
                 self.assertIsInstance(testcase, task.ApiTestCase)

tests/test_utils.py

@@ -12,9 +12,9 @@ class TestUtils(ApiServerUnittest):
 
     def test_load_json_testcases(self):
         testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/simple_demo_no_auth.json')
+            os.getcwd(), 'tests/data/demo_testset_hardcode.json')
         testcases = utils.load_testcases(testcase_file_path)
-        self.assertEqual(len(testcases), 2)
+        self.assertEqual(len(testcases), 3)
         testcase = testcases[0]["test"]
         self.assertIn('name', testcase)
         self.assertIn('request', testcase)
@@ -23,9 +23,9 @@ class TestUtils(ApiServerUnittest):
 
     def test_load_yaml_testcases(self):
         testcase_file_path = os.path.join(
-            os.getcwd(), 'tests/data/simple_demo_no_auth.yml')
+            os.getcwd(), 'tests/data/demo_testset_hardcode.yml')
         testcases = utils.load_testcases(testcase_file_path)
-        self.assertEqual(len(testcases), 2)
+        self.assertEqual(len(testcases), 3)
         testcase = testcases[0]["test"]
         self.assertIn('name', testcase)
         self.assertIn('request', testcase)
@@ -45,28 +45,28 @@ class TestUtils(ApiServerUnittest):
 
         # absolute file path
         path = os.path.join(
-            os.getcwd(), 'tests/data/simple_demo_no_auth.json')
+            os.getcwd(), 'tests/data/demo_testset_hardcode.json')
         testset_list = utils.load_testcases_by_path(path)
         self.assertEqual(len(testset_list), 1)
-        self.assertEqual(len(testset_list[0]["testcases"]), 2)
+        self.assertEqual(len(testset_list[0]["testcases"]), 3)
         testsets_list.extend(testset_list)
 
         # relative file path
-        path = 'tests/data/simple_demo_no_auth.yml'
+        path = 'tests/data/demo_testset_hardcode.yml'
         testset_list = utils.load_testcases_by_path(path)
         self.assertEqual(len(testset_list), 1)
-        self.assertEqual(len(testset_list[0]["testcases"]), 2)
+        self.assertEqual(len(testset_list[0]["testcases"]), 3)
         testsets_list.extend(testset_list)
 
         # list/set container with file(s)
         path = [
-            os.path.join(os.getcwd(), 'tests/data/simple_demo_no_auth.json'),
+            os.path.join(os.getcwd(), 'tests/data/demo_testset_hardcode.json'),
-            'tests/data/simple_demo_no_auth.yml'
+            'tests/data/demo_testset_hardcode.yml'
         ]
         testset_list = utils.load_testcases_by_path(path)
         self.assertEqual(len(testset_list), 2)
-        self.assertEqual(len(testset_list[0]["testcases"]), 2)
+        self.assertEqual(len(testset_list[0]["testcases"]), 3)
-        self.assertEqual(len(testset_list[1]["testcases"]), 2)
+        self.assertEqual(len(testset_list[1]["testcases"]), 3)
         testsets_list.extend(testset_list)
         self.assertEqual(len(testsets_list), 4)
 
@@ -81,7 +81,7 @@ class TestUtils(ApiServerUnittest):
         # absolute folder path
         path = os.path.join(os.getcwd(), 'tests/data')
         testset_list_1 = utils.load_testcases_by_path(path)
-        self.assertGreater(len(testset_list_1), 6)
+        self.assertGreater(len(testset_list_1), 5)
 
         # relative folder path
         path = 'tests/data/'