update function doc strings

ate/runner.py

@@ -16,6 +16,7 @@ class TestRunner(object):
         """ create/update variables binds
         @param config_dict
             {
+                "name": "description content",
                 "requires": ["random", "hashlib"],
                 "function_binds": {
                     "gen_random_string": \
@@ -30,11 +31,6 @@ class TestRunner(object):
                     {"random": {"func": "gen_random_string", "args": [5]}},
                 ]
             }
-        @return variables binds mapping
-            {
-                "TOKEN": "debugtalk",
-                "random": "A2dEx"
-            }
         """
         requires = config_dict.get('requires', [])
         self.context.import_requires(requires)
@@ -49,15 +45,7 @@ class TestRunner(object):
 
     def parse_testcase(self, testcase):
         """ parse testcase with variables binds if it is a template.
-        """
-        self.pre_config(testcase)
-
-        parsed_testcase = self.testcase_parser.parse(testcase)
-        return parsed_testcase
-
-    def run_test(self, testcase):
-        """ run single testcase.
-        @testcase
+        @param (dict) testcase
             {
                 "name": "testcase description",
                 "requires": [],  # optional, override
@@ -66,6 +54,30 @@ class TestRunner(object):
                 "request": {},
                 "response": {}
             }
+        @return (dict) variables binds mapping
+            {
+                "TOKEN": "debugtalk",
+                "random": "A2dEx"
+            }
+        """
+        self.pre_config(testcase)
+
+        parsed_testcase = self.testcase_parser.parse(testcase)
+        return parsed_testcase
+
+    def run_test(self, testcase):
+        """ run single testcase.
+        @param (dict) testcase
+            {
+                "name": "testcase description",
+                "requires": [],  # optional, override
+                "function_binds": {}, # optional, override
+                "variable_binds": {}, # optional, override
+                "request": {},
+                "response": {}
+            }
+        @return (tuple) test result of single testcase
+            (success, diff_content)
         """
         testcase = self.parse_testcase(testcase)
 
@@ -82,42 +94,61 @@ class TestRunner(object):
         success = False if diff_content else True
         return success, diff_content
 
-    def run_testsets(self, testsets):
-        """ run testcase suite.
-        @testsets
-            [
-                {
+    def run_testset(self, testset):
+        """ run single testset, including one or several testcases.
+        @param (dict) testset
+            {
+                "name": "testset description",
+                "config": {
                     "name": "testset description",
-                    "config": {
-                        "requires": [],
-                        "function_binds": {},
-                        "variable_binds": []
+                    "requires": [],
+                    "function_binds": {},
+                    "variable_binds": []
+                },
+                "testcases": [
+                    {
+                        "name": "testcase description",
+                        "variable_binds": {}, # override
+                        "request": {},
+                        "response": {}
                     },
-                    "testcases": [
-                        {
-                            "name": "testcase description",
-                            "variable_binds": {}, # override
-                            "request": {},
-                            "response": {}
-                        },
-                        testcase12
-                    ]
-                },
-                {
-                    "name": "XXX",
-                    "config": {},
-                    "testcases": [testcase21, testcase22, testcase23]
-                },
+                    testcase12
+                ]
+            }
+        @return (list) test results of testcases
+            [
+                (success, diff_content),    # testcase1
+                (success, diff_content)     # testcase2
             ]
         """
         results = []
 
-        for testset in testsets:
-            config_dict = testset.get("config", {})
-            self.pre_config(config_dict)
-            testcases = testset.get("testcases", [])
-            for testcase in testcases:
-                result = self.run_test(testcase)
-                results.append(result)
+        config_dict = testset.get("config", {})
+        self.pre_config(config_dict)
+        testcases = testset.get("testcases", [])
+        for testcase in testcases:
+            result = self.run_test(testcase)
+            results.append(result)
 
         return results
+
+    def run_testsets(self, testsets):
+        """ run testsets, including one or several testsets.
+        @param testsets
+            [
+                testset1,
+                testset2,
+            ]
+        @return (list) test results of testsets
+            [
+                [   # testset1
+                    (success, diff_content),    # testcase11
+                    (success, diff_content)     # testcase12
+                ],
+                [   # testset2
+                    (success, diff_content),    # testcase21
+                    (success, diff_content)     # testcase22
+                ]
+            ]
+        """
+        return [self.run_testset(testset) for testset in testsets]

ate/utils.py

@@ -147,8 +147,8 @@ def load_testcases_by_path(path):
             - list/set container with file(s) and/or folder(s)
     @return testcase sets list, each testset is corresponding to a file
         [
-            {"config": {}, "testcases": [testcase11, testcase12]},
-            {"config": {}, "testcases": [testcase21, testcase22, testcase23]},
+            {"name": "desc1", "config": {}, "testcases": [testcase11, testcase12]},
+            {"name": "desc2", "config": {}, "testcases": [testcase21, testcase22, testcase23]},
         ]
     """
     if isinstance(path, (list, set)):
@@ -169,6 +169,7 @@ def load_testcases_by_path(path):
 
     if os.path.isfile(path):
         testset = {
+            "name": "",
             "config": {},
             "testcases": []
         }
@@ -178,6 +179,7 @@ def load_testcases_by_path(path):
             for key in item:
                 if key == "config":
                     testset["config"] = item["config"]
+                    testset["name"] = item["config"].get("name", "")
                 elif key == "test":
                     testset["testcases"].append(item["test"])
 

test/test_runner.py

@@ -76,16 +76,30 @@ class TestRunner(ApiServerUnittest):
             }
         )
 
-    def test_run_testcase_suite_json_success(self):
+    def test_run_testset_json_success(self):
         testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.json')
         testsets = utils.load_testcases_by_path(testcase_file_path)
-        results = self.test_runner.run_testsets(testsets)
+        results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
         self.assertEqual(results, [(True, {}), (True, {})])
 
-    def test_run_testcase_suite_yaml_success(self):
+    def test_run_testsets_json_success(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.json')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testsets(testsets)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, {}), (True, {})])
+
+    def test_run_testset_yaml_success(self):
+        testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results, [(True, {}), (True, {})])
+
+    def test_run_testsets_yaml_success(self):
         testcase_file_path = os.path.join(os.getcwd(), 'test/data/simple_demo_no_auth.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
-        self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, {}), (True, {})])

test/test_runner_v2.py

@@ -31,22 +31,37 @@ class TestRunnerV2(ApiServerUnittest):
         success, _ = self.test_runner.run_test(testcase)
         self.assertTrue(success)
 
-    def test_run_testcase_auth_suite_yaml(self):
+    def test_run_testset_auth_yaml(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'test/data/simple_demo_auth_hardcode.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results, [(True, {}), (True, {})])
+
+    def test_run_testsets_auth_yaml(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'test/data/simple_demo_auth_hardcode.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, {}), (True, {})])
+
+    def test_run_testset_auth_json(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'test/data/simple_demo_auth_hardcode.json')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
         self.assertEqual(len(results), 2)
         self.assertEqual(results, [(True, {}), (True, {})])
 
-
-    def test_run_testcase_auth_suite_json(self):
+    def test_run_testsets_auth_json(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'test/data/simple_demo_auth_hardcode.json')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
-        self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, {}), (True, {})])
 
     def test_run_testcase_template_yaml(self):
         testcase_file_path = os.path.join(
@@ -57,10 +72,18 @@ class TestRunnerV2(ApiServerUnittest):
         success, _ = self.test_runner.run_test(testcases[1]["test"])
         self.assertTrue(success)
 
-    def test_run_testcase_template_sets_yaml(self):
+    def test_run_testset_template_yaml(self):
+        testcase_file_path = os.path.join(
+            os.getcwd(), 'test/data/demo_template_sets.yml')
+        testsets = utils.load_testcases_by_path(testcase_file_path)
+        results = self.test_runner.run_testset(testsets[0])
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results, [(True, {}), (True, {})])
+
+    def test_run_testsets_template_yaml(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'test/data/demo_template_sets.yml')
         testsets = utils.load_testcases_by_path(testcase_file_path)
         results = self.test_runner.run_testsets(testsets)
-        self.assertEqual(len(results), 2)
-        self.assertEqual(results, [(True, {}), (True, {})])
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0], [(True, {}), (True, {})])