change testcase data structure, in order to distinguish config and test

test/data/demo_binds.yml

@@ -1,38 +1,38 @@
 -
-  variable_binds:
-    - TOKEN: "debugtalk"
+    variable_binds:
+        - TOKEN: "debugtalk"
 
 -
-  variable_binds:
-    - var: [1, 2, 3]
+    variable_binds:
+        - var: [1, 2, 3]
 
 -
-  variable_binds:
-    - data: {'name': 'user', 'password': '123456'}
+    variable_binds:
+        - data: {'name': 'user', 'password': '123456'}
 
 -
-  variable_binds:
-    - TOKEN: "debugtalk"
-    - token: $TOKEN
+    variable_binds:
+        - TOKEN: "debugtalk"
+        - token: $TOKEN
 
 -
-  function_binds:
-    add_one: "lambda x: x + 1"
-    add_two_nums: "lambda x, y: x + y"
-  variable_binds:
-    - add1: {"func": "add_one", "args": [2]}
-    - sum2nums: {"func": "add_two_nums", "args": [2, 3]}
+    function_binds:
+        add_one: "lambda x: x + 1"
+        add_two_nums: "lambda x, y: x + y"
+    variable_binds:
+        - add1: {"func": "add_one", "args": [2]}
+        - sum2nums: {"func": "add_two_nums", "args": [2, 3]}
 
 -
-  requires:
-    - random
-    - string
-    - hashlib
-  function_binds:
-    gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
-    gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
-  variable_binds:
-    - TOKEN: debugtalk
-    - random: {"func": "gen_random_string", "args": [5]}
-    - data: "{'name': 'user', 'password': '123456'}"
-    - authorization: {"func": "gen_md5", "args": [$TOKEN, $data, $random]}
+    requires:
+        - random
+        - string
+        - hashlib
+    function_binds:
+        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
+        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
+    variable_binds:
+        - TOKEN: debugtalk
+        - random: {"func": "gen_random_string", "args": [5]}
+        - data: "{'name': 'user', 'password': '123456'}"
+        - authorization: {"func": "gen_md5", "args": [$TOKEN, $data, $random]}

test/data/demo_template_separate.yml

@@ -1,62 +1,62 @@
 
--
-  name: create user which does not exist
-  requires:
-    - random
-    - string
-    - hashlib
-  function_binds:
-    gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
-    gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
-  variable_binds:
-    - TOKEN: debugtalk
-    - random: {"func": "gen_random_string", "args": [5]}
-    - data: '{"name": "user", "password": "123456"}'
-    - authorization: {"func": "gen_md5", "args": ["$TOKEN", "$data", "$random"]}
-    - expected_status_code: 201
-  request:
-    url: http://127.0.0.1:5000/api/users/1000
-    method: POST
-    headers:
-      Content-Type: application/json
-      authorization: "${authorization}"
-      random: "${random}"
-    data: "${data}"
-  response:
-    status_code: "${expected_status_code}"
-    headers:
-      Content-Type: application/json
-    body:
-      success: true
-      msg: user created successfully.
+- test:
+    name: create user which does not exist
+    requires:
+        - random
+        - string
+        - hashlib
+    function_binds:
+        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
+        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
+    variable_binds:
+        - TOKEN: debugtalk
+        - random: {"func": "gen_random_string", "args": [5]}
+        - data: '{"name": "user", "password": "123456"}'
+        - authorization: {"func": "gen_md5", "args": ["$TOKEN", "$data", "$random"]}
+        - expected_status_code: 201
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: "${authorization}"
+            random: "${random}"
+        data: "${data}"
+    response:
+        status_code: "${expected_status_code}"
+        headers:
+            Content-Type: application/json
+        body:
+            success: true
+            msg: user created successfully.
 
--
-  name: create user which does not exist
-  requires:
-    - random
-    - string
-    - hashlib
-  function_binds:
-    gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
-    gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
-  variable_binds:
-    - TOKEN: debugtalk
-    - random: {"func": "gen_random_string", "args": [5]}
-    - data: '{"name": "user", "password": "123456"}'
-    - authorization: {"func": "gen_md5", "args": ["$TOKEN", "$data", "$random"]}
-    - expected_status_code: 500
-  request:
-    url: http://127.0.0.1:5000/api/users/1000
-    method: POST
-    headers:
-      Content-Type: application/json
-      authorization: "${authorization}"
-      random: "${random}"
-    data: "${data}"
-  response:
-    status_code: "${expected_status_code}"
-    headers:
-      Content-Type: application/json
-    body:
-      success: true
-      msg: user created successfully.
+- test:
+    name: create user which does not exist
+    requires:
+        - random
+        - string
+        - hashlib
+    function_binds:
+        gen_random_string: "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))"
+        gen_md5: "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
+    variable_binds:
+        - TOKEN: debugtalk
+        - random: {"func": "gen_random_string", "args": [5]}
+        - data: '{"name": "user", "password": "123456"}'
+        - authorization: {"func": "gen_md5", "args": ["$TOKEN", "$data", "$random"]}
+        - expected_status_code: 500
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: "${authorization}"
+            random: "${random}"
+        data: "${data}"
+    response:
+        status_code: "${expected_status_code}"
+        headers:
+            Content-Type: application/json
+        body:
+            success: true
+            msg: user created successfully.

test/data/simple_demo_auth_hardcode.json

@@ -1,53 +1,57 @@
 [
     {
-        "name": "create user which does not exist",
-        "request": {
-            "url": "http://127.0.0.1:5000/api/users/1000",
-            "method": "POST",
-            "headers": {
-                "content-type": "application/json",
-                "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
-                "random": "A2dEx"
+        "test": {
+            "name": "create user which does not exist",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/1000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json",
+                    "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
+                    "random": "A2dEx"
+                },
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
             },
-            "json": {
-                "name": "user1",
-                "password": "123456"
-            }
-        },
-        "response": {
-            "status_code": 201,
-            "headers": {
-                "Content-Type": "application/json"
-            },
-            "body": {
-                "success": true,
-                "msg": "user created successfully."
+            "response": {
+                "status_code": 201,
+                "headers": {
+                    "Content-Type": "application/json"
+                },
+                "body": {
+                    "success": true,
+                    "msg": "user created successfully."
+                }
             }
         }
     },
     {
-        "name": "create user which existed",
-        "request": {
-            "url": "http://127.0.0.1:5000/api/users/1000",
-            "method": "POST",
-            "headers": {
-                "content-type": "application/json",
-                "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
-                "random": "A2dEx"
+        "test": {
+            "name": "create user which existed",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/1000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json",
+                    "authorization": "a83de0ff8d2e896dbd8efb81ba14e17d",
+                    "random": "A2dEx"
+                },
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
             },
-            "json": {
-                "name": "user1",
-                "password": "123456"
-            }
-        },
-        "response": {
-            "status_code": 500,
-            "headers": {
-                "Content-Type": "application/json"
-            },
-            "body":{
-                "success": false,
-                "msg": "user already existed."
+            "response": {
+                "status_code": 500,
+                "headers": {
+                    "Content-Type": "application/json"
+                },
+                "body":{
+                    "success": false,
+                    "msg": "user already existed."
+                }
             }
         }
     }

test/data/simple_demo_auth_hardcode.yml

@@ -1,39 +1,39 @@
--
-  name: create user which does not exist
-  request:
-    url: http://127.0.0.1:5000/api/users/1000
-    method: POST
-    headers:
-      Content-Type: application/json
-      authorization: a83de0ff8d2e896dbd8efb81ba14e17d
-      random: A2dEx
-    json:
-      name: "user1"
-      password: "123456"
-  response:
-    status_code: 201
-    headers:
-      Content-Type: application/json
-    body:
-      success: true
-      msg: user created successfully.
+- test:
+    name: create user which does not exist
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: a83de0ff8d2e896dbd8efb81ba14e17d
+            random: A2dEx
+        json:
+            name: "user1"
+            password: "123456"
+    response:
+        status_code: 201
+        headers:
+            Content-Type: application/json
+        body:
+            success: true
+            msg: user created successfully.
 
--
-  name: create user which existed
-  request:
-    url: http://127.0.0.1:5000/api/users/1000
-    method: POST
-    headers:
-      Content-Type: application/json
-      authorization: a83de0ff8d2e896dbd8efb81ba14e17d
-      random: A2dEx
-    json:
-      name: "user1"
-      password: "123456"
-  response:
-    status_code: 500
-    headers:
-      Content-Type: application/json
-    body:
-      success: false
-      msg: user already existed.
+- test:
+    name: create user which existed
+    request:
+        url: http://127.0.0.1:5000/api/users/1000
+        method: POST
+        headers:
+            Content-Type: application/json
+            authorization: a83de0ff8d2e896dbd8efb81ba14e17d
+            random: A2dEx
+        json:
+            name: "user1"
+            password: "123456"
+    response:
+        status_code: 500
+        headers:
+            Content-Type: application/json
+        body:
+            success: false
+            msg: user already existed.

test/data/simple_demo_no_auth.json

@@ -1,50 +1,54 @@
 [
     {
-        "name": "create user which does not exist",
-        "request": {
-            "url": "http://127.0.0.1:5000/api/users/1000",
-            "method": "POST",
-            "headers": {
-                "content-type": "application/json"
+        "test": {
+            "name": "create user which does not exist",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/1000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json"
+                },
+                "cookies": {},
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
             },
-            "cookies": {},
-            "json": {
-                "name": "user1",
-                "password": "123456"
-            }
-        },
-        "response": {
-            "status_code": 201,
-            "headers": {
-                "Content-Type": "application/json"
-            },
-            "body": {
-                "success": true,
-                "msg": "user created successfully."
+            "response": {
+                "status_code": 201,
+                "headers": {
+                    "Content-Type": "application/json"
+                },
+                "body": {
+                    "success": true,
+                    "msg": "user created successfully."
+                }
             }
         }
     },
     {
-        "name": "create user which existed",
-        "request": {
-            "url": "http://127.0.0.1:5000/api/users/1000",
-            "method": "POST",
-            "headers": {
-                "content-type": "application/json"
+        "test": {
+            "name": "create user which existed",
+            "request": {
+                "url": "http://127.0.0.1:5000/api/users/1000",
+                "method": "POST",
+                "headers": {
+                    "content-type": "application/json"
+                },
+                "json": {
+                    "name": "user1",
+                    "password": "123456"
+                }
             },
-            "json": {
-                "name": "user1",
-                "password": "123456"
-            }
-        },
-        "response": {
-            "status_code": 500,
-            "headers": {
-                "Content-Type": "application/json"
-            },
-            "body":{
-                "success": false,
-                "msg": "user already existed."
+            "response": {
+                "status_code": 500,
+                "headers": {
+                    "Content-Type": "application/json"
+                },
+                "body":{
+                    "success": false,
+                    "msg": "user already existed."
+                }
             }
         }
     }

test/data/simple_demo_no_auth.yml

@@ -1,4 +1,4 @@
--
+- test:
     name: create user which does not exist
     request:
         url: http://127.0.0.1:5000/api/users/1000
@@ -16,7 +16,7 @@
             success: true
             msg: user created successfully.
 
--
+- test:
     name: create user which existed
     request:
         url: http://127.0.0.1:5000/api/users/1000

test/test_runner.py

@@ -14,10 +14,18 @@ class TestRunner(ApiServerUnittest):
         url = "http://127.0.0.1:5000/api/users"
         return requests.delete(url)
 
-    def test_run_single_testcase_success(self):
+    def test_run_single_testcase_yaml_success(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml')
+        testcases = utils.load_testcases(testcase_file_path)
+        testcase = testcases[0]["test"]
+        success, _ = self.test_runner.run_test(testcase)
+        self.assertTrue(success)
+
+    def test_run_single_testcase_json_success(self):
         testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.json')
         testcases = utils.load_testcases(testcase_file_path)
-        success, _ = self.test_runner.run_test(testcases[0])
+        testcase = testcases[0]["test"]
+        success, _ = self.test_runner.run_test(testcase)
         self.assertTrue(success)
 
     def test_run_single_testcase_fail(self):

test/test_runner_v2.py

@@ -20,14 +20,16 @@ class TestRunnerV2(ApiServerUnittest):
         testcase_file_path = os.path.join(
             os.getcwd(), 'test/data/simple_demo_auth_hardcode.yml')
         testcases = utils.load_testcases(testcase_file_path)
-        success, _ = self.test_runner.run_test(testcases[0])
+        testcase = testcases[0]["test"]
+        success, _ = self.test_runner.run_test(testcase)
         self.assertTrue(success)
 
     def test_run_single_testcase_json(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'test/data/simple_demo_auth_hardcode.json')
         testcases = utils.load_testcases(testcase_file_path)
-        success, _ = self.test_runner.run_test(testcases[0])
+        testcase = testcases[0]["test"]
+        success, _ = self.test_runner.run_test(testcase)
         self.assertTrue(success)
 
     def test_run_testcase_auth_suite_yaml(self):
@@ -50,7 +52,7 @@ class TestRunnerV2(ApiServerUnittest):
         testcase_file_path = os.path.join(
             os.getcwd(), 'test/data/demo_template_separate.yml')
         testcases = utils.load_testcases(testcase_file_path)
-        success, _ = self.test_runner.run_test(testcases[0])
+        success, _ = self.test_runner.run_test(testcases[0]["test"])
         self.assertTrue(success)
-        success, _ = self.test_runner.run_test(testcases[1])
+        success, _ = self.test_runner.run_test(testcases[1]["test"])
         self.assertFalse(success)

test/test_utils.py

@@ -17,22 +17,24 @@ class TestUtils(ApiServerUnittest):
             os.getcwd(), 'test/data/simple_demo_no_auth.json')
         testcases = utils.load_testcases(testcase_file_path)
         self.assertEqual(len(testcases), 2)
-        self.assertIn('name', testcases[0])
-        self.assertIn('request', testcases[0])
-        self.assertIn('response', testcases[0])
-        self.assertIn('url', testcases[0]['request'])
-        self.assertIn('method', testcases[0]['request'])
+        testcase = testcases[0]["test"]
+        self.assertIn('name', testcase)
+        self.assertIn('request', testcase)
+        self.assertIn('response', testcase)
+        self.assertIn('url', testcase['request'])
+        self.assertIn('method', testcase['request'])
 
     def test_load_yaml_testcases(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'test/data/simple_demo_no_auth.yml')
         testcases = utils.load_testcases(testcase_file_path)
         self.assertEqual(len(testcases), 2)
-        self.assertIn('name', testcases[0])
-        self.assertIn('request', testcases[0])
-        self.assertIn('response', testcases[0])
-        self.assertIn('url', testcases[0]['request'])
-        self.assertIn('method', testcases[0]['request'])
+        testcase = testcases[0]["test"]
+        self.assertIn('name', testcase)
+        self.assertIn('request', testcase)
+        self.assertIn('response', testcase)
+        self.assertIn('url', testcase['request'])
+        self.assertIn('method', testcase['request'])
 
     def test_parse_response_object_json(self):
         url = "http://127.0.0.1:5000/api/users"